{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[AMP WARNING][Frontend.cpp:121][1657891302:36191]Sleep 0.1s waiting for AMP Server socket: /tmp/harp/ccl_ipc_socket/system/server.socket\n",
      "[AMP INFO][Frontend.cpp:152][1657891302:136464]pid=232, start to allocate gpu resource ...\n",
      "Reading lines...\n",
      "Load 65080 sentence pairs\n",
      "Counting words...\n",
      "['will', 'you', 'come', 'round', 'next', 'tuesday', '?'] ['next', 'tuesday', '?', 'just', 'a', 'moment', '.', 'it', 'will', 'be', 'the', 'twenty-first', 'of', 'may', '?', 'oh', ',', 'dear', '.', 'hum', ',', 'i', 'promised', 'my', 'aunt', 'i', '’', 'll', 'be', 'at', 'her', 'birthday', 'party', '.']\n",
      "Counted words:\n",
      "dialog 21609\n",
      "(tensor([126,  28,  26, 597, 191, 669,  14,   3,   1,   1,   1,   1,   1,   1,\n",
      "          1,   1,   1,   1,   1,   1,   1,   1,   1,   1,   1]), tensor(8), tensor([  191,   669,    14,   173,    33,   864,    25,    69,   126,   105,\n",
      "           43, 15610,    23,   255,    14,   160,     6,  2698,    25,  2919,\n",
      "            6,    15,  3217,   215,  5543]), tensor(25))\n",
      "Reading lines...\n",
      "Load 8975 sentence pairs\n",
      "Reading lines...\n",
      "Load 8974 sentence pairs\n"
     ]
    }
   ],
   "source": [
    "!python data.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from config import *\n",
    "from transformer import TransformerEncoder, TransformerDecoder, EncoderDecoder\n",
    "from attention import sequence_mask\n",
    "from data import loaddata, MyDataset, Lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Timer:\n",
    "    \"\"\"Record multiple running times.\"\"\"\n",
    "    def __init__(self):\n",
    "        self.times = []\n",
    "        self.start()\n",
    "\n",
    "    def start(self):\n",
    "        \"\"\"Start the timer.\"\"\"\n",
    "        self.tik = time.time()\n",
    "\n",
    "    def stop(self):\n",
    "        \"\"\"Stop the timer and record the time in a list.\"\"\"\n",
    "        self.times.append(time.time() - self.tik)\n",
    "        return self.times[-1]\n",
    "\n",
    "    def avg(self):\n",
    "        \"\"\"Return the average time.\"\"\"\n",
    "        return sum(self.times) / len(self.times)\n",
    "\n",
    "    def sum(self):\n",
    "        \"\"\"Return the sum of time.\"\"\"\n",
    "        return sum(self.times)\n",
    "\n",
    "    def cumsum(self):\n",
    "        \"\"\"Return the accumulated time.\"\"\"\n",
    "        return np.array(self.times).cumsum().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import time\n",
    "from IPython import display\n",
    "    \n",
    "def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):\n",
    "    \"\"\"Set the axes for matplotlib.\"\"\"\n",
    "    axes.set_xlabel(xlabel)\n",
    "    axes.set_ylabel(ylabel)\n",
    "    axes.set_xscale(xscale)\n",
    "    axes.set_yscale(yscale)\n",
    "    axes.set_xlim(xlim)\n",
    "    axes.set_ylim(ylim)\n",
    "    if legend:\n",
    "        axes.legend(legend)\n",
    "    axes.grid()\n",
    "\n",
    "class Animator:\n",
    "    \"\"\"For plotting data in animation.\"\"\"\n",
    "    def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,\n",
    "                 ylim=None, xscale='linear', yscale='linear',\n",
    "                 fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,\n",
    "                 figsize=(3.5, 2.5)):\n",
    "        # Incrementally plot multiple lines\n",
    "        if legend is None:\n",
    "            legend = []\n",
    "#         use_svg_display()\n",
    "        self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)\n",
    "        if nrows * ncols == 1:\n",
    "            self.axes = [self.axes,]\n",
    "        # Use a lambda function to capture arguments\n",
    "        self.config_axes = lambda: set_axes(self.axes[\n",
    "            0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)\n",
    "        self.X, self.Y, self.fmts = None, None, fmts\n",
    "\n",
    "    def add(self, x, y):\n",
    "        # Add multiple data points into the figure\n",
    "        if not hasattr(y, \"__len__\"):\n",
    "            y = [y]\n",
    "        n = len(y)\n",
    "        if not hasattr(x, \"__len__\"):\n",
    "            x = [x] * n\n",
    "        if not self.X:\n",
    "            self.X = [[] for _ in range(n)]\n",
    "        if not self.Y:\n",
    "            self.Y = [[] for _ in range(n)]\n",
    "        for i, (a, b) in enumerate(zip(x, y)):\n",
    "            if a is not None and b is not None:\n",
    "                self.X[i].append(a)\n",
    "                self.Y[i].append(b)\n",
    "        self.axes[0].cla()\n",
    "        for x, y, fmt in zip(self.X, self.Y, self.fmts):\n",
    "            self.axes[0].plot(x, y, fmt)\n",
    "        self.config_axes()\n",
    "        display.display(self.fig)\n",
    "        display.clear_output(wait=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "def loss_fn(out, tar):\n",
    "    out = out.contiguous().view(-1, out.shape[-1])\n",
    "    tar = tar.contiguous().view(-1)\n",
    "    return F.cross_entropy(out, tar, ignore_index=2)  # pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_clipping(net, theta):\n",
    "    \"\"\"Clip the gradient.\"\"\"\n",
    "    if isinstance(net, nn.Module):\n",
    "        params = [p for p in net.parameters() if p.requires_grad]\n",
    "    else:\n",
    "        params = net.params\n",
    "    norm = torch.sqrt(sum(torch.sum((p.grad**2)) for p in params))\n",
    "    if norm > theta:\n",
    "        for param in params:\n",
    "            param.grad[:] *= theta / norm\n",
    "            \n",
    "            \n",
    "def train_seq2seq(net, data_iter, valid_iter, lr, num_epochs, lang, device, first_train = True, min_ppl=1e9, i = 70, lloss=-1,lppl=-1):\n",
    "   # \"\"\"训练序列到序列模型\"\"\"\n",
    "    def xavier_init_weights(m):\n",
    "        if type(m) == nn.Linear:\n",
    "            nn.init.xavier_uniform_(m.weight)\n",
    "        if type(m) == nn.GRU:\n",
    "            for param in m._flat_weights_names:\n",
    "                if \"weight\" in param:\n",
    "                    nn.init.xavier_uniform_(m._parameters[param])\n",
    "    \n",
    "    net.to(device)\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr, betas=(0.9, 0.98), eps=1e-9)\n",
    "    net.train()\n",
    "    animator = Animator(xlabel='epoch', ylabel='loss',\n",
    "                     xlim=[0, num_epochs])\n",
    "    animator1 = Animator(xlabel='epoch', ylabel='ppl',\n",
    "                     xlim=[0, num_epochs])\n",
    "    LOSS = PPL = 0\n",
    "    \n",
    "    timer = Timer()\n",
    "    \n",
    "    if first_train:\n",
    "        net.apply(xavier_init_weights)\n",
    "    else:\n",
    "        net.eval()\n",
    "        PPL_AVG = 0\n",
    "        LOSS_AVG = 0\n",
    "        with torch.no_grad():\n",
    "#             i = 0\n",
    "            for batch in valid_iter:\n",
    "                # 转device\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                # inputs带<eos>不带<bos>，outputs带<bos>不带<eos>\n",
    "                # 初始bos\n",
    "                bos = torch.tensor([2  # lang.word2index['<bos>']\n",
    "                                    ] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)\n",
    "                # Teacher forcing, 输入X和正确答案，希望得到正确答案\n",
    "                # Y_hat\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                # Y_hat和Y中都有eos，没有bos\n",
    "                l = loss_fn(Y_hat, Y)\n",
    "                ppl = torch.exp(l)\n",
    "                LOSS_AVG += l.item()\n",
    "                PPL_AVG += ppl.item()\n",
    "            PPL_AVG /= len(valid_iter)\n",
    "            min_ppl = PPL_AVG\n",
    "            LOSS_AVG /= len(valid_iter)\n",
    "            if lloss == -1:\n",
    "                lloss = LOSS_AVG\n",
    "            if lppl == -1:\n",
    "                lppl = PPL_AVG\n",
    "            animator1.add(0, (lppl, PPL_AVG,))\n",
    "            animator.add(0, (lloss, LOSS_AVG,))\n",
    "                \n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        net.train()\n",
    "#         lr = 512**(-0.5) * i**(-0.5)\n",
    "        \n",
    "        for batch in data_iter:\n",
    "            optimizer.zero_grad()\n",
    "            X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "            bos = torch.tensor([2] * Y.shape[0],\n",
    "                          device=device).reshape(-1, 1)\n",
    "            dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 强制教学\n",
    "            Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "            l = loss_fn(Y_hat, Y)\n",
    "            ppl = torch.exp(l)\n",
    "            l.backward()\n",
    "#             grad_clipping(net, 20)\n",
    "            optimizer.step()\n",
    "            with torch.no_grad():\n",
    "                LOSS += l.item()\n",
    "                PPL += ppl.item()\n",
    "        i += 1\n",
    "#             if i == 30:\n",
    "#                 break\n",
    "        LOSS /= len(data_iter)\n",
    "        PPL /= len(data_iter)\n",
    "\n",
    "        \n",
    "        net.eval()\n",
    "        PPL_AVG = 0\n",
    "        LOSS_AVG = 0\n",
    "        with torch.no_grad():\n",
    "#             i = 0\n",
    "            for batch in valid_iter:\n",
    "                # 转device\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                # inputs带<eos>不带<bos>，outputs带<bos>不带<eos>\n",
    "                # 初始bos\n",
    "                bos = torch.tensor([2  # lang.word2index['<bos>']\n",
    "                                    ] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)\n",
    "                # Teacher forcing, 输入X和正确答案，希望得到正确答案\n",
    "                # Y_hat\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                # Y_hat和Y中都有eos，没有bos\n",
    "                l = loss_fn(Y_hat, Y)\n",
    "                ppl = torch.exp(l)\n",
    "                LOSS_AVG += l.item()\n",
    "                PPL_AVG += ppl.item()\n",
    "#                 i += 1\n",
    "#                 if i == 10:\n",
    "#                     break\n",
    "                    \n",
    "            PPL_AVG /= len(valid_iter)\n",
    "            LOSS_AVG /= len(valid_iter)\n",
    "            if PPL_AVG < min_ppl:\n",
    "                if min_ppl != 1e9:\n",
    "                    torch.save(encoder, MODEL_ROOT + \"trans_encoder.mdl\")\n",
    "                    torch.save(decoder, MODEL_ROOT + \"trans_decoder.mdl\")\n",
    "                min_ppl = PPL_AVG\n",
    "                \n",
    "        if (epoch + 1) % 1 == 0:\n",
    "            \n",
    "            animator1.add(epoch + 1, (PPL, PPL_AVG,))\n",
    "            animator.add(epoch + 1, (LOSS, LOSS_AVG,))\n",
    "            \n",
    "    print(f'loss {LOSS:.3f}, PPL {PPL:.3f}, {timer.stop()/60:.1f} '\n",
    "        f'min on {str(device)}')\n",
    "    print(f'loss {LOSS_AVG:.3f}, PPL {PPL_AVG:.3f} ')\n",
    "    print(lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = torch.load(DATA_ROOT + \"dataset\")\n",
    "lang = torch.load(DATA_ROOT + \"dialog.lang\")\n",
    "valid_data = torch.load(DATA_ROOT + \"validdata\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter = loaddata(dataset, 64)\n",
    "valid_iter = loaddata(valid_data, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = TransformerEncoder(\n",
    "        len(lang), key_size, query_size, value_size, num_hiddens,\n",
    "        norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n",
    "        num_layers, dropout)\n",
    "decoder = TransformerDecoder(\n",
    "        len(lang), key_size, query_size, value_size, num_hiddens,\n",
    "        norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,\n",
    "        num_layers, dropout)\n",
    "net = EncoderDecoder(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = torch.load(MODEL_ROOT + \"trans_encoder1{}.mdl\".format(num_examples))\n",
    "decoder = torch.load(MODEL_ROOT + \"trans_decoder1{}.mdl\".format(num_examples))\n",
    "net = EncoderDecoder(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 3.893, PPL 50.160, 7.8 min on cuda:0\n",
      "loss 3.950, PPL 53.138 \n",
      "0.004972233026098313\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC4CAYAAAAsVToeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAHfNJREFUeJzt3Xl4lPW5//H3PTNZZ7KRhBAgJAQSQIIIURSOtlHReq62LrX1tNX+bNVqqx7E9pzWWq3a2h6Xy5banp5KrbXW2tZaq55Tq3VLcUFZFJAdAgQIhi2E7NvM/ftjJiGEQBYyeWYy9+u65srMPMvcCXye7/P9zrOIqmKMiT0upwswxjjDwm9MjLLwGxOjLPzGxCgLvzExysJvTIyy8BsToyz8xsQoC78xMcrCb0yM8jhdwMlIT0/XyZMnO11GrxobG/F6vU6X0SurbXAiuTaAlStXHlDV7P7OH9Xhz8nJYcWKFU6X0avy8nLKysqcLqNXVtvgRHJtACJSOZD5bbffmBhl4TcmRln4jYlRFn4TEVSV3Q/vhmvgUPkhp8uJCRZ+ExEOPH+ArbdshWpYc8Eaqv6nyumSRjwLv3GUBoJXksq6OItpf5gGT0PGhRlsuXELu3+22+HqRjYLv3HMofJDLC9ZTktlC+IScj6fAz6Y8cIMCu8rJOeLOU6XOKJZ+M2wU1V2/3Q3q+evBoVAW+Co6eIWJnx7AnGZcQTaAqy/cj31q+odqnbksvCbYeVv9rPxyxvZunArmZ/KZPZ7s0kuSj7u/K27Wqn9Zy0fzPuAfX/eN4yVjnwWfjOsKu+tZO8Teym4p4CSZ0vwpJ74INOkSUmUrijFd5qP9VesZ/ud27vGCczJierDe030CHQEcHlcTPjOBNI/ns6oC0f1e9mEMQmc9sZpbL5xM5X3VtJxuIOih4vCWG1ssPCbsFJVqn5WRfVvq5m1ZBYen2dAwe/kSnAx5dEp+Gb5SC9LD0Olscd2+03YdPXvb9lKwvgE1H9yu+siwvibx+Mr8aGqVHyrgkOvxfYBQarK3qf2cvjtwwNe1sJvwqJlVwsfnPNBsH9/dwElf+27fz8Q/jo/B188yOpPrGb3z3YTq3eeqvxBJRuu3MCeR/YMeFkLvwmLTdduonlzMyXPl1BwVwHikiFdvyfNw+yls8n8ZCZbF2xl01c3EWgN9L3gCNC2v43m7c0AjLlmDMW/Kmbqb6YOeD3W5zdDRlXRdsUV76L4kWICrQG8U8N38QtPioeSv5aw464dVN5bSVtVGzNenIHI0G5oIkWgPcCeX+xh+13bSZ2Tysx/zCRxfCJjrxs7qPVZ+M2Q8Lf42fy1zfjr/Ux/ZjpJE5OG5XPFJUz8wUS8M7y4El0jNvg1r9aw9ZatNK1vIuPCDCYvOvkrWFn4zUlr2dXCus+so35FPfl35YMCw5zB0VeM7npe9csqPOme4OHCI0D1k9Vs/NJGEgsTKXm+hMxPZw7JRs7Cb05K7ZJa1n1uHYHmACXPlZB1SZaj9WhA2f/0fmrfqKVxdSMT752IuKNvb8Df6KdlZwveaV6yLs1i0o8nMfbrY3EnuofsM2zAzwyav8XP+i+sx5PuYfZ7sx0PPgS7Aae+dCq5X81l5307+fCSD+mo63C6rH5TVfb+YS/Lpi5j7aVrUb/i8XnIuzVvSIMP1vKbQfC3+HHFu3AnupnxfzNIKkzCkxY5/5U6Bxx9M31suWUL7897n9IVpUMenqFW/0E9Wxds5fBbh/HN9lH0cFFY91oi51/MRIXO/n3mpzIpuKuAlFkpTpfUKxFh3E3jSD4lmYbVDREf/Nq3aln1sVXEZcVR/Kticr+SG/buiu32m36rXVLLytNX0rSpCe/MyL1+fXcZ52aQtzAPgEOvH2LXT3ZFzAFBgfYADWsaAEibm0bh/YXM2TyHsdeNHZZxCgu/6ZOqUvXfVaw+f3VX/z770n7fGyJi7H1qLxXfqGDTNc4fEFTzag0rTlvBqrJVdBzuCF7D4D8nEJceN2w12G6/6VPzlma2LtzKqItGMe3JaRHVvx+IKYunkJiXyI67d9C0sYnpz04nITdhWGto3tZMxTcrOPDcARILE5n6m6m4U53pkkTnv6IZFh0NHXh8HpKLk5n19ixSTk8Z8sN0h5O4hIK7CvCWeNlw9QZWnr6S2e/NJnF84rB8fvOOZpadsgxxCxN/OJHx3xjv6FiE7fabXtW+Wct7k99j/7P7AUidkxrVwe8u+/JsZr8zm9H/NpqEceFt+VWVhtXBfn1SQRKT7p/EnE1zyL893/FBSAu/OUpX//681XjSPCRPO/4ltqKZ71Qfk388GRGhuaKZ7XdtP+lTjnuq/6CeVR9fFRwk3doEwPhbxg/bnkZfLPymi7/Fz6ZrN7Hl5i1kfCKD0mWleKdFx6j+ydj/zH4qv1/Jh5/+kPba9pNeX9uBNjZ9bRMrS1fStKGJol8UDdu5DgNhfX7TpeZvNVT/ppr8O/MpuHvoT8ONVBO+PQFPhoctN23h/TPfZ8YLM0ieMrg9Hn+jn+XTl9N+sJ1xC8ZRcFcBcRnDN4I/EGEJv4ikAAuA7UAZ8ICqbu02fW7o/f1AQFUf6zYtFXhaVS8KR23mWO217cSlx5F9eTal75dG7IE74TT2+rEkT01m3eXrWHnmSma+OpPU01P7vXzDmgZ8p/pwe90U/qiQ1LmpeE+J7L2mcO32ZwNLVPUp4Anghh7Tbye4QXgUOFNEcgEkeKrS5YBdo3kYqCpVv6ji3YJ3afgwOCgVi8HvlP6xdEpXlJJ1SVa/g9u8o5m1l69lxcwVXfcYzL02N+KDDyDhPtpJRL4E1Krq/3Z77xVVvSD0/CZgl6q+ICIXA8uA+1T1y8dZ3/XA9QDZ2dmlTz/9dFjrH6yGhgZ8Pp/TZfSqoaEBX7wPFgF/B84E7gAioNyI+rs1AU8BXwISetTWDPwB+CPgBq4ErgDiHakUgHPPPXelqp7e3/nD1ucXkTHAbQT/hN/rMbleRHJV9SMgLjT/KQQ3EtUnOldZVRcDiwGmTJmiZWVlYaj+5JWXlxOxtf25nJQfpFC/rJ78O/IpuCdy+veR9Hfb98w+1v9+PSmbUyj5awlLtyylrKwMVWXFrBU0rm5k9BdHU3h/YcSM4A9E2MKvqtXAQhHJBx4Ebu02+WbgRhGpBi4EbgI+B7hEZB4wQ0RuAx5W1aZw1RhL2va3IXESPHz0OWha38T0v0wn+zPRd5jucBn92dG4nnOx4aoNrDxjJXwV9BxF3EL+HfnEj4kn/ezovYx4uAb8ZgP7VHU3UA/MFJF4AFVtU9U9wJ0ikgNMUdXtwAPdlp+qqveFo7ZYEGgLULe0jrplddQvr6duWR2tla0U/byIcTeNg69A6d2lJ7xNlgnKuiSLWUtnsfaStfB9qM6vJveaXEZ/dnTfC0e4cLX8VcBVInIQKCU4wLeA4MWdHgQQkWLgauC7YaohJgRag2eG1S+vJz43nuzLsgm0BFh17ipQSCxIJPXMVFJuTiH9vFAr5cGCPwC+Eh+ly0p5+4dvk3WZ8xcsGSphCb+q7gUeCr18PPTz3c7pIlICpAN3aC8jjscb7DNHVNxWQe3rtTSsbkDbgn/C7Cuyyb4sG0+qh5mvz8Q73Ut8toMjUCNIXGYcXEzEfmc/GI4c5KOqa5343GiiqrRUtlC/vJ76ZfXULa8DYFb5LCDYZ3d73YxfOJ6UM1JInZNKQt6R49QzyjIcqdtEj6g+wq/DHwzJSLhcc9v+NhpWNTDqguB97DZdt4nqx6oBkHjBd5qPtHlpXb/vjBdmOFmuGQGiOvyBnbAkcQmeDA+edA9xGXGkzktl8kPBa5rv/tluAq0B4jLi8KR78GR4SMhL6OrvOrnhaNrcxIHnDwRb9uX1tOxoAWBu1VwSxiaQ/blsUk5PIeWMFHyn+nDF22kYZmhFdfjrk5SaT6Vwis9LR20HHYc6jjoza9dDu2itbD1qmazPZFHylxIA3sl5B23X4MYj9Mi6OIvxC8YDsPOBnbhT3F0bjriMOBLyE0gY0//TQANtoQG5ZcGQ530rD+80L3Xv1rHtW9tInJhIypwUxt40ltQ5qcG+JZB5UebJ/nmMOaGoDn97inLv1IO8+e1TSY4/9lc5a/tZ+Ov9XRuG9kPtR12FZtzN42jf3077ofauefz1fiAY2m3f3nbMOvP+I49JD06io66Dd8a+07XH0bn3MebqMWRfng07YeWZK2lYdWRALi47juwrsoPXYr8si3n/Os8G5Ixjojr8GYkuDja28dt3Kvl62aRjposInlRP8O6wE45dvuB7BcddtyvexTnN59BxqKNrw9BR20HChCOt/tgbxh41vXVXKx21oWvEp4MrycX4W7oNyE1I6OpmeFI8ELuH0ZsIENXhT3DD3CnZPLKkgqvOmkBK4tB+DeNOdOPOdfd6nTdPqqdrbKFXqUdG5o2JRFE/inTr/GJqm9r57Ts7nC7FmKgS9eGfmZfO/GmjWbxkG3UtJ38VFmNiRdSHH2Dh/GLqWjp47K3tTpdiTNQYEeEvGZfGRdPH8Os3t3O4yVp/Y/pjRIQfYOEFRdS3dvDoW8d+PWeMOdaICf/UMal88tRcHntrO4ca25wux5iIN2LCD7Dw/CKa2v0sftNaf2P6MqLCX5STwsUzx/L42zs40NDa9wLGxLARFX6ABecX0drh55F/VjhdijERbcSFf1K2j0tnjeOJpZXsq2txuhxjItaICz/AgvOK6Ago/2OtvzHHNSLDX5Dl5fLZ4/j9ezupPmytvzG9GZHhB/j384oIBJRflG/te2ZjYtCIDX/eqGSuOCOPPy7bRVVts9PlGBNx+hV+EfmKiJwiIjNE5E8i8uUw1zUkbjo3eMrtf79hrb8xPfW35T+squuBbwPXAlFxb6Jx6Ul8fk4eTy/fxa4au/GPMd31N/xpInIWsFNVG4CDYaxpSN1YNhmXS/j569b6G9Ndf8O/GrgIuF9ETgNyw1fS0BqTlsiVZ07gmfd3s+NAo9PlGBMx+hV+VX1fVe9W1cNAI/DL8JY1tL5eNok4t/Dw61ucLsWYiNHfAb+7QwN+1wFfB+4Mb1lDa3RKIl86K5/nPqiiYn+D0+UYExH6u9u/BtgIXKSq3wCirgN9w8cnkeBx8/Br1vobA/0P/0TgZ8DTIuIjCi86neVL4Op5Bbyweg9b9tY7XY4xjutv+H8CPKiqTwNjgOXhKyl8rv9YIclxbhZZ629Mv8OvwMdF5MfAvwDLwldS+IzyxnPN2RP525qP2Fhd53Q5xjiqv+H/D6AeWAwcBv4zbBWF2XVnF5KS4GHRK9b6m9jW3/DvUtVnVXWjqj4HVJ1oZhFJEZHvisgXRWSxiEzuMX2uiHxHRK4TkWtC76WLyDdF5CoR+Z2IFA/uVzqxtOQ4rj1nIi+tq2Zt1eFwfIQxUaG/4ff1eO3tY/5sYImqPgU8AdzQY/rtwAOq+ihwpojkAtOAf6jqk8BjwPX9rG3Arjl7IqmJHha9ujlcH2FMxOvvvfo+EpHHgZ1AHvDsiWZW1W1A51U0JwJLesySqKr+0PM1wBmq+kK36ZnAqt7WLSLXE9owZGdnU15e3s9f4Wjz84RnN+zjsedfozDNPah1nEhDQ8Ogaws3q21wIrm2QVHVfj2ANKAUSAc+3Y/5xwCLgB8Bnh7TngVyQ88XAhd3m5YM/BRI6OsziouLdbDqW9p15j0v69WPvTfodZzIG2+8EZb1DgWrbXAiuTZVVWCF9jPPqnr8ll9E7gdmExzpP2pSKNj/28dGpRpYKCL5wIPArd0m3wzcKCLVwIXATaHPlNB896hqWC+/60vwcMPHJnH/SxtZWXmI0vyMcH6cMRHnRH3+F1X1AlW9sMfjAvroj4vIbBEZH3pZD8wUkXgRiQdQ1T2qeifwDLBVVTtvsnct8Liq1pzcr9U//29uPpneeOv7m5h03JZfVf95gmlL+1hvFXCViBwk2FW4HVhAcK/hQYDQaP7VwHdDrz8DfAE4O7gDQKuq9hwoHFLeBA9f+/gkfvjiBpZtr2HOxFHh/DhjIkp/B/wGRFX3Ag+FXj4e+vlu53QRKSE4dnBHqK+Cqj5LHwOJ4XDVWfk8smQbP3llM3+4/qzh/nhjHOPINfxUda2qvtUZfCclxbu5sWwSS7cd5J2KA06XY8ywGbEX8ByIL545gZzUBBa9soUI2B4ZMyws/EBinJubz53Msh01vL01aq5QZsxJsfCHXHFGHmPTEnnolU3W+puYYOEPSfC4ufm8Ij7YWUv55v1Ol2NM2Fn4u/ls6XjGZyTxk1c2W+tvRjwLfzfxHhcLzitize7DvLZhn9PlGBNWFv4eLps9jvzMZH5srb8Z4Sz8PcS5g63/+o/qeHndXqfLMSZsLPy9uOS0sRRmeVn06mYCAWv9zchk4e+Fx+3ilvlFbKyu5+9rq50ux5iwsPAfx6dOHUvRaB+LXt2M31p/MwJZ+I/D7RIWzi9my74G/m/NHqfLMWbIWfhP4F9LxjB1TAo/fXULHf6A0+UYM6Qs/CfgCrX+2w408vwqa/3NyGLh78MnpucwfWwqD7++hXZr/c0IYuHvg4hw6/xiKg828df3T3i7AmOiioW/H86fNppTx6fx8OtbaOuw1t+MDBb+fhARbr2gmN2Hmnlm5W6nyzFmSFj4+6msOJtZE9L5+etbaO3w972AMRHOwt9PIsI3Lihmz+EWnl6+y+lyjDlpFv4BOHtyFmcUZPDzN7bS0m6tv4luFv4BCLb+U9hb18pT7+10uhxjToqFf4DmTspkbmEmvyivoLnNWn8TvSz8g3DrBcUcaGjlyXcrnS7FmEGz8A/CnImjOKcoi1/+s4LG1g6nyzFmUCz8g7RwfjEHG9t4Yqm1/iY6WfgHqTQ/g7Ip2TyypIL6lnanyzFmwCz8J+HW+cXUNrXz23d2OF2KMQNm4T8JM/PSmT9tNIuXbKPOWn8TZSz8J2nh/GLqWjp47K3tTpdizIBY+E9Sybg0PjE9h1+/uZ3apjanyzGm38ISfhFJEZHvisgXRWSxiEzuMX2uiHxHRK4TkWu6vX+ziFwpIneKyKhw1BYOC+cXU9/awaNvWutvoke4Wv5sYImqPgU8AdzQY/rtwAOq+ihwpojkishUIEdVfw88DdwaptqG3LTcVD55ai6/eXs7NY3W+pvo4AnHSlV1G7At9HIisKTHLImq2nls7BrgDCAHWBlafpOIzO5t3SJyPXA9QHZ2NuXl5UNb/CDNSw3wYpufO54s54op8TQ0NERMbT1ZbYMTybUNRljCDyAiY4DbgCbgez0m14tIrqp+BMSF3ssCKrrNE0cvVHUxsBhgypQpWlZWNpRln5RlDR/wj3V7+f6Vc1m7YimRVFt35eXlVtsgRHJtgxG2AT9VrVbVhcAjwIM9Jt8M3CgiNwEXAh8C+4B0ABERIOr2nxecX0Rrh59H/lnR98zGOCxcA36zRWR86GU9MFNE4kUkHkBV96jqncAzwFZV3Q68CZweWuYUYEU4agunSdk+Lp01jieWVlLbYtf6M5EtXC1/FfBvIvJl4B6CA3wLgFs6ZxCR4tB73wVQ1c3AntAynwN+GqbawmrBeUV0BJTnK9rtgh8mooVrwG8v8FDo5eOhn+92TheREoK7+HeoqnZb7ufhqGc4FWR5+ezs8fxpxS6mfe8lxmckMTHLR2GWl8JsL4VZPiZme8lNTcTlEqfLNTEsbAN+J6Kqa5343OHy/Uunk9G+j4SsCWw/0Mi2Aw2s3FFDY7eLfyTGuSjI9DIp20dhtpeJWV4Ks31MzPKSltTrWKcxQ8qR8I90CR43Z+V6KCsr7npPVdlX30rF/obgBmF/I9sPNLJuz2FeWld91J2As3zxwY1BaC+hc69hwigv8R47KNMMDQv/MBERclITyUlNZN6krKOmtXUE2FnTFNooHNk4vLZxHwdWtHbN53YJeRlJR+0ldHYlclITCH5JYkz/WPgjQLzHxeTRPiaP9hE81umIw83t7Ah1Hbbtb2RbaMOwdNtBWtqPfKOQHO8+aqMwKbRRKMhKJiXRuhHmWBb+CJeWFMfMvHRm5qUf9X4goFTXtXTtLXRuFFbvquVva/bQrRfB6JSErg1DYZaXw9Ud6MZ9eNxCnNsVekiPny48biG+2/M4l8sGKUcQC3+UcrmEselJjE1P4l8mH92NaO3ws/NgExWhcYXOrsTL66qPnHuwavmgPtfjkuNsNI4897hdxPd47nG5iPOE5nG5iPP0vo5dO9o5kLKbUd44RnkTGJUcT4Y3Dl+Cx7o1Q8zCPwIleNwU5aRQlJNyzLTapjZeePVNZs4qpd0foM0foMOvtPsDoUffzzv8Stsxz5WO0Htt3Z43tvm7np9o3d0HPP+4afUxdce5hYzkeEZ5g48Mb3xowxDPqOS44M9uj4zkeBLj3GH9O0c7C3+MSU+OZ0Kq+5huhNMCAaU9EODVN5ZQMnsONY1tHGpqo6axnZrGVmoa2znU2EZNUxuHGtvY8FEdNY1tHG5uR7X3dSbHu4/aYHRuFEZ547o2Ht03JulJcXjcw/9tiqrSHtqItrb7afMHaOsIPlo7Ase+7gjQ5vcfM89AWfhNRHC5hASXG2+ckJ/pJT/T26/lOvwBDje3d9tQdG40go/uG4xtBxqoaWg76niLntKS4kIbiVC3o9uG4qNd7ex6tzIUuCPha+sloEdC6+91etfz0PtOsPCbqOZxu8j0JZDpS+j3Mi3tfmqbjt1QdH99qKmNqtpm1lYdpqax7UjLuu7o49M8LiHe4wo+3C4S4oI/4z1u4j0uEtwukuM9pB8z/cgjIfQ6IbRM57p6zhNc1n3U+/Fdy7pIun+Af7uBzW5M9EuMczMmzc2YtMR+za+qNLb5efn1JZxz9jwSPG4SPMEBSncUf/th4TemDyKCL8FDZpKL0Sn922BEAztW1JgYZeE3JkZZ+I2JURZ+Y2KU6PGOkIgCIlIPbHK6juPIAg44XcRxWG2DE8m1AUxR1WMP6zyOaB/t36Sqp/c92/ATkRVW28BZbYMnIgO67qXt9hsToyz8xsSoaA//YqcLOAGrbXCstsEbUH1RPeBnjBm8aG/5zQgjIi4ROV1EZjpdS7SQQV7lJGrDH6m38xaRdBH5pohcJSK/C92cJKKISKqIvOR0HT2F7u/4LWC/qh57RQ+HiIhbRP5TRL4kIotEZEIE1OQK/R97DpgXem+iiNwhIl8Rkc/3tY6oDH+E3857GvAPVX0SeIzQHYUjRaiVuJzgvREjRqiu7wGLVLXS6Xp6mAHEqervgKeAyxyuB1UNhP6PrQI6W/47gIdV9TfA+SJywiu2RGX4gXPodjtvoNfbeTtBVZeq6oehl5kE/3EiyaeBvwORdjPBMqAd+LKI/EBEcvqYfzhtBM4RkXHALOBFh+s5nhxVrQs93wycdqKZozX8WUBdt9cRd21qEUkmuJH6s9O1dBKRU4BaVa12upZenAZ8oKq/JHgD10UO19NFVVuAHwBPAoXAdmcrOq74bs8PA9knmjlawx/Rt/MO1XQrcI+qtvY1/zD6FDBPRG4DZojIbaGNVCTo6PZ8DZDnVCE9icg04FRVPRd4DXjA4ZKOp3sOMoC9J5o5WsMf6bfzvhZ4XFVrnC6kO1V9QFXvU9X7gA9Dz5ucrivkTY5030bTx3/cYXYhwV1/gFeIoA1TD9UikhZ6XkQfXc6oPLZfVTeLSOftvAuIoNt5i8hngC8AZ4e+gWlV1RucrSryqeoqEdkpIl8BcoHvOF1TN38EvhsaaB4F/NLhejq7lZ8ESoFEEYkn2DW5VUSqgFe79f97X4cd5GNMbIrW3X5jzEmy8BsToyz8xsQoC78xMcrCb0yMsvCbsBORMhGJ1ANjYpaF34SdqpYTQedfmCALvzExKiqP8DPhIyILgFogH1hC8PRVN8GzKIuAnar6iIj4gAVABZADvKeq74lINsHTmDcDk4BfqerB0Lo/QfCsuIuBi/o6As2El4XfdBGR84AkVX04dHLSE8CvgY+p6uOheZ4RkWeAbwB/UNW1oXmfF5FLgP8CblfVfaFTYMcABwnuZa5W1ZdFJADMBV4e7t/RHGHhN911Hid+KcHWvvOiGt3P/a8guFcwE7gbQFVVRJqBNCBfVfeF3q8CqjrX0e1U4n30cbqpCT/r85vuNgD1qvqcqv6FULg5cqUYCO7KbwM+BIqh6xTmJILnkB8QkaLOmUUkdRjqNoNgJ/aYLqEQf5/gRSF2Eww4wNeANwieQblBVZ8IhXoBwQ1BDvCuqi4NXd/uR8AWoAb4EzAB+C3w7wRPM/0h4Ae+o6r1w/PbmZ4s/OaERKQMOFtV73W6FjO0bLffHJeIeAleGfYUEZnkdD1maFnLb0yMspbfmBhl4TcmRln4jYlRFn5jYpSF35gYZeE3Jkb9f1ZlVLEoOlPyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPYAAAC4CAYAAADQQBHUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAHwBJREFUeJzt3Xl8XHW9//HXZyaZbNOsnaZLkiZd0qbpvlGghUbWK+pPFGSRqyAVld8VLuKCXFREuXrdLuLvXkELKhTFimyuKFKgxTYlaZrue5a2aZs0IWn2yWQ+vz/OJE3bpJ1MMpkl3+fjMY/MTM7M+TSd95ztc75HVBXDMKKLLdQFGIYx/EywDSMKmWAbRhQywTaMKGSCbRhRyATbMKKQCbZhRCETbMOIQibYhhGFTLANIwrFhLqA80lNTdVp06aFuox+tba2kpSUFOoy+mVqC0w411ZaWnpSVV3+Th/Wwc7MzKSkpCTUZfTrzTffZOXKlaEuo1+mtsCEc20iUjWY6c2quGFEIRNsw4hCJtiGEYXCehvbiB6eUx62f2g7pIGuUMQuoS4pqpkltjEi7GPsqEfhZdj9yd14Pd5QlxTVTLCNoNFupfKRSlr3tCIiLNywEO6E2udq2X3LbrxdJtzBYoJtBIX7pJtt799G5TcqqX2+9vQvboOpP5hK3Qt1VDxYEboCo5zZxjaG3anNp9h5w07ctW7yf57PhDsnnPH77Puzic2MJf2a9BBVGP3MEtsYVo1vNVK2ogyxCwvfWcjEVRMROXdH2fjbxuNwOfC6vVQ8XEF3a3cIqo1eJtjGsBpz0RgmfX4Si0oXMWbRmAtO3/ROE1XfqmLbv2zDc8ozAhWODibYxpC17W9j+4e309XYhT3ezrQfTCM2Pdav16YVpTHr17No+mcT5VeX09XYFeRqRwcTbGNI6l6uo3RxKU3rm2jf3x7Qe4y7aRyFvyukZUsL5VeU01Vvwj1UJthGQLweLwe/cpCd1+8kcUYii7csJnlJcsDv57rexeyXZtN5tJOOwx3DWOnoZPaKGwE59MAhjvzwCBM/N5Fp/z0NW9zQlxEZ12Ww7NAy7Il2ADwtHmKc5iMaCLPENgal55JQ2fdnU/BcAfn/mz8soe7RE+rDPzpM6cJSs/QOkAm24RdV5fBjh9l+3Xa0W4mbEEfmrZlBm1/yxcm4T7jZetlW2isC23YfzUywjQvyNHvYdfMuDt53EHEI3o7gt4KmXJzCvH/Mw9PkYevlW2nb3xb0eUYTE2zjvFp3tbJl6RbqXqhjyn9NYfZLs7En2Udk3smLk5m/bj7edi9bi7biaTHHuf1l9kwYA1KvsvOGnXQ1dDHv9XmkFaWNeA3OeU7mvzWf5pJmsyNtEMxfyjiH1+0FAVusjYJfF+BwOYibFBeyepJmJZE0yxpksOH1BmLTYv3qaosW2j34a9ibVXHjDJ1HO9latJVDXz4EwJj5Y0Ia6r60WzlwzwG2XrGVpk1NoS4n6Lrbu6n+r2o2z9o86NeaYBu93lv3HiULS2gpbyH54sCbTYJF7MLcv84ldmws267aRuP6xlCXFBTqVY49dYzi6cUceuAQifmJg36PoK2Ki8iLQIPvYSlgByqBZUClqq4O1ryNwVFVDn/vMIcetD5E89+cT1JBeI6vHZ8Tz4K3F1B+RTnbrt3GnFfnkHbFyG/7B5XAsaePEZcVx6xfzyL1slQY5EhSwVxiv6qqq3y3nwKnVPWPwNeAO4M4X2OQOg51UPlwJa4bXSzcvDBsQ90jbmIc89+cT8KUBOperAt1OcOi6Z9NlF9TjvuEGxFhzqtzWLhxoRXqAARz51mhiNwHJAE/UdVnfM+nAeVBnK/hp44jHcRnxZMwNYFFWxaRODOx33Onw5Ej08H89fOJGWN9hL1uLzZH5G1Ztu5ppeKrFZx8+SSO8Q7a9rXhyHQQm+Hf2XEDkZ4WwWARkQXA3ar6aRG5A/gYsEpVjw4w/V3AXQAul2vR2rVrg1pfoFpaWnA6naEuo19+1fY34EfA/cBVI1CUT1D+bnXAF7DWA1cG/jYj+n/qBR4D/gTEAzcDNwAJ/U9eVFRUqqqL/X5/VQ36DXi9z3078AwQf6HX5efna7hat25dqEsY0Plq6+7o1r2f3avrWKdlK8u083jnyBWmwfm7dTV1aemlpbrOtk6PPXss4PcZif/T7o7u3vu7PrlL9927TztrL/x/AJToIDIXlHUXESkSkULf/TSgUUSu9n2RdANxQE4w5m0MrKOqg7LlZdQ8UUP2V7KZ+/e5ODIdoS5ryGKSY5j717mkXp7Knk/s4djTx0Jd0jm8nV4OP3aYjTkbadnWAsDMX8xk+mPTcbiG//8gWNvYZcCnRGQ2MBNrRekjIhIHLML69tkXpHkbA2guaaZtfxuFLxXi+rDfF26MCDHOGOb8aQ47rt/B3jv3Yh9jZ9yN40JdFupVan9TS8VDFXRUdpB2VRoSa+3HCOb+jKAEW1Ubsbbg+nrM9/MPfr/PsFU0eqlXaS5tJnlJMq6PukhdmTrkHTPhyp5gZ84rc6h4qIK0K0N/CEy9StllZZx65xTOBU7m/mwu6VeNzMisYb0b0VODOWVvCLrqu9j+ge2UXVrWe3ZUtIa6hy3OxtTvTyU2LZbujm5OPH9ixGto3d2KqiI2YdyN4yh4roBFJYtGLNQQ5sGOaYHN+ZvZ97l9dB7tDHU5EeVUySlKFpXw3j/eY9rj00iYNsDu1ihW89Madt+ym8pvVvYOEBFM7Yfa2XXrLt6d9S4Nf7Z6s7LuzSLz1kzENrKHEcP6JJAjLi96ZSrHnjrG8V8eJ+sLWUx5dEqoywp/f4Syn5ThyHSwYP0CkpeGX3voSMi6J4vWba1UPlyJt9NL3qN5Qdmudde5qfpWFTVP1CCxwuSHJpOyImXY5zMYYR1sYuDpKzt58tGlVD1ShbpPf+t6mj29zQmGRVWtD24tpK5MpeC5AhxjI3+vd6DELsx4agYSJ1R/pxpvh5epP5w6rOFWr1K2ooz2A+1MWDWB3G/kEjch9CfNhPWq+BiH8Pa+OqoTPMz8xUymfM9aWr/3j/fYmL2Rym9X4mke3Sffq1c5+ceTbL1yK7W/8V0j65Mw989zR3Woe4hNyP9pPpPuncTxXx0flk06b5eX4786jrfLi9iE6T+ZzpIdS5jxxIywCDWEebCTHUJ8rI2nN1gXb+v5pnVMdJB6eSqVX6ukeEoxh394mO720XWJGE+zhyOPH2HzjM3s+OAO2va0oV7fGo0dc/3pPkSEaf89jcXli4nPireaOLyD3+ZWVep+X8e7s99lz+17OPnSSQDSr0onaWZ49deHdbBtAh9dmMVLW49ysuX0N21SQRJzXpnDwuKFOBc4OfjFg5RdUjYiO0jCxbZrtnHg3gPEumKZ9fwsllUsY/xt40NdVtgSEeKz4gGo/GYlez65Z1DX6G58u5EtF29h5w07kRhh9quzcd0Yvr0AYb+R+qnleTxXXM2aTVX8+5X5Z/wueWky8/42j8a3Guk62YWI4PV4qXuhDtcNLmwxYf295TdVpfGNRmp+VsOM1TOIGRND3rfzsDvto3bH2FDY4mycWHMCb6eXgucKsMWe/3Oiqhy49wBdJ7uY8fQMxn9ifNivEYV9sKe6nLxv5jie3VjFZy+fSnzsuQPppV5++tS2+lfqrUMcD1eS9808XDe6RvxQw3Dpbu/mxJoTHH38KK07Wol1xdK6s5WUZSmkvS/0DRiRavJXJ2OLt3HwCwfxur0U/rbwnLHRO6o7qP5ONXmP5hGbHkvhC4U4JjqwJ4zMQI5DFRGLtFXL86hvdfPK1n5PCDvD2I+MpfDFQiRG2HXzLkoWlHDy1ZMRt5ruPuFmY/ZG9t21D+ww4xczWFa9jJRloT2MEi2y78tm+v9Mp/6VenbesLP389H1XhcHv3yQ4vxijj19jKZ/WkMwJUxNiJhQQwQssQEunppBwYRkVq+v4GOLs897uEJEcF3vYuyHxlL721oqv1HJoQcPkXFdhnVeWZhSVU5tOkVLeQuTPjsJR6aDiZ+ZSPrV6aRclhIx50lHkkl3T8IWZ0Mcvr/t81B8fTGeJg+Z/5pJ3iN5xE+OD22RAYqIYIsIq5bncf/vynl7/0kuz7/wTguxC5m3ZuL6mIvOI52IXfA0edj9id1kfzGb1BWBjUwx3LxuL3W/q+PIj4/Q/G4zsZmxjL99PPZ4u2nGGQET7pxw+sEO6wokU747Befc8DzX3l8RsSoO8MF5Exk3Jo7V6w8N6nW2GBsJuVY7ZdveNpo3N7P1sq2UX1vOqXdPBaNUvzX8rYFNuZvYfdtuPKc8TP/f6Vx04CLs8WG8ahHNvm4d/4/0UEMEBdsRY+OTl+Syfv9J9h5vDug9kpcmc9HBi5jy/Sk0lzSzZekWdly/g+6OkTsG3ry1mZbt1vm4CVMTcM5zMucvc1i6aymTPjfJDIofSlHUzxMxwQa4dWnOGQ0rgbAn2sn5Yg7LKpaR+0guEie9S8iuhuBccN3r8VL3Yh1ll5dRuqCUym9WAlaw5/5lLhnXZkTsnnsjPEVUsNOSHL0NK3XNQ2sNjBkTQ+7Xcil8vhCA9sp2NmZtZM+qPXRUDd+lW2tW11A8rZidH91JR1UHU38wlRmrZwzb+xtGfyIq2GA1rLg9XtZsqhrW97U77Uy4awInnj1B8fRi9v3bPjqPBfbl0ba3rberyV3jJn5yPIW/L+SiAxeRfX82sanRfU60EXoRF+ypLidXzBzHmk1VdHQN37axY6yD6Y9ZO6/G3zGeY08eY/PMzXia/DvJRL1K/V/qKb+2nM0zN3PyZauPePJDk1nw1gJcH4meTjgj/EXkJ+3OFVbDystlF25YGaz47HhmPDmDpXuWMu3H04hJsXZmHX/2eL8h93Z5Ofo/R9k8azPb37+d1vJWch/J7R3o3Ww7G6EQkbtgL56SwawJyazeUMFNS87fsBKohKkJJEz1HSbb18aeT+whJi2G7C9lk3VPFvhGbBK7cOTxI8SkxFCwpgDXja6IHLjeiC4R+QkUEVatyONAbQtv7Qv+JV4S8xNZVLqI5EuSqXiwgo3ZG+FW6G7rRmzCgncWsLB4IZkfzzShNsJCxH4KPzDXalh5agiHvgZjzMIxzP3jXBa8s4C0q9Pg/aBdVn+xY6zDtHwaYSVigz0cDSuBSLkkxTpE9ml6t78NI9xEbLDhdMPKUxsG12ZqGNEuooOdluTghkVZvFxWM+SGFcOIJhEdbIBPXZqHu9vLs8PcsGIYkSzigz3F5eTKguFvWDGMSBa0vT8i8iLQ4HtYivUlchC4Gtinqk8M17zuXD6F13++iZfKjnLLUnMRT8MI5hL7VVVdpaqrgJ8D7ar6V6xLra8azhktm5LOrAnJPLWhIuKGQDKMYAhmsAtF5D4ReQhIUtWnfc/HAMN68HmkG1YMI9xJIEs4EVmqqpv9nHYBcLeqftr3+F5gg6qWDjD9XcBdAC6Xa9HatWv9qsnjVb74VjuTnMKXlgT/AnQtLS04neE50oapLTDhXFtRUVGpqi72+wWq2u8NuBv4O/C3s25/B7YP9LoB3ut1388rgX/x93X5+fk6GP/vjf06+St/1N3Hmgb1ukCsW7cu6PMIlKktMOFcG1Cig8jc+VbFW1T1KlW9+qzbVcDXz/dlISJFIlLou58GHPctuRNV9S9+f+sM0scvyiEh1s5T60emzdQwwtWAwVbVZ/o+FpE0EZkvIsmq+tIF3rcMuEZEbgLuAf4DeBL4iIj80ndbMtTiz5aaaDWsvLK1htrm4RsFxTAijV+Hu3zbvVdhHa6aKiKvqOqagaZX1UbgR2c9vTTgKgfhjktzeXZTFWs2VvGFq80QRMbo5O9e8RxVvVFVH1DVG4FZwSxqKHobVoqrTcOKMWr5G+x9Zz2uABCRvOEtZ3jcuXwKDa1uXgrCCCuGEQn87TxbLCI5gBfry2CWiLiwlty3Bau4QC2bkk7hRKth5abF2djM8ETGKOPvErsbeBvY4Pv5JPAOsDNIdQ3JGQ0r+03DijH6+BvsX2Ad134Iqx20UlXfAn4crMKG6ro5E8lMjjOHvoxRyd9gfxb4vKpeDXwJ+DKAqrYFq7Ch6hlhZcOBk+w+FtprdBnGSPM32OWqWgegqieAvQAiEheswobDrUuthpWhXBLIMCKRv8E+JSJZIjJRRCYBHhGZCNwUxNqGzDSsGKOVv3vF78ZqUOm7e3kJkAM80+8rwsQdl+ayptg0rBiji7/Bvk9VS85+UkSmD3M9w26Ky8kVMzN5dlMVdxdNIz7WXHvaiH5+rYr3F2rf8/uHt5zgWLUij/faunhxi2lYMUaHiB/zzB8X5aUze1IyT204hNdrRlgxot+oCLaIsGr5FA7WtZoRVoxRYVQEG+D9cyZYDSvm0JcxCoyaYJuGFWM0GTXBhtMNK2apbUS7URXs1EQHNy7O4pWtR6k9ZRpWjOg1qoINcMeleXi8ai4JZES1URfsvLFJXFmQyZpNVbS7zQgrRnQadcEGWLXc17BSdiTUpRhGUIzKYC/1Naw8vaHCNKwYUWlUBts0rBjRblQGG6yGlfHJ8azecCjUpRjGsBu1we5pWHnnQD27akzDihFdRm2wwTSsGNFrVAc7JTGWjy3O4tVy07BiRJcRD7aIhNUg3z0NK89sNA0rRvTwdwSVQRORF4EG38NSrAv1XQlMxBpqKSzk+hpWniuu4v8WTSPBYUZYMSJfMJfYr6rqKt/tp6q6CVgDOII4z4CYhhUj2gQz2IUicp+IPCQiKUGcz5AtzUtnzqQUnjINK0aUENXgfpB9F7y/W1U/LSK5wEOquuo8098F3AXgcrkWrV27Nqj19dhY4+HJbZ38+8I45o+78BZKS0sLTqdzBCobPFNbYMK5tqKiolJVXez3C1Q16Dfgdd/PXGC1v6/Lz8/XkeL2dOtFj76ut/xso1/Tr1u3LrgFDYGpLTDhXBtQooPIXFBWxUWkSEQKfffTgOPBmM9wirXbuP3SXP55sJ6dNU2hLscwhiRY29hlwDUichNwD/CgiCwHPgYUiMjNIpIcpHkH7JYlOSQ6TMOKEfmCcrhLVRuBH531dDXWZXi/F4x5DoeUxFhuXJTFrzdX88C1MxmXHB/qkgwjIKO686w/pmHFiAYm2GfJHZvEVQWZrCk2I6wYkcsEux+rVkyhsa2L328xDStGZDLB7seS3DTmZqWYEVaMiGWC3Q8R4c7leRw62cq6vbWhLscwBs0EewDvnzOBCSnxrF5vDn0ZkccEewCxdmuElY2HTMOKEXlMsM/DNKwYkcoE+zysEVay+UN5DSfMCCtGBDHBvoA7Ls31NaxUhroUw/CbCfYFTM5I4upZmTxXXE2b2xPqcgzDLybYfjjdsHI01KUYhl9MsP2weLJpWDEiiwm2H3oaVipMw4oRIUyw/WQaVoxIYoLtp1i7jdt9DSs7jpqGFSO8mWAPws1LrYaVp03DihHmTLAHISXBalh5tbyG+nZvqMsxjAEF7Uog0epTl+bxzMZK7n+rne+X/YOcjEQmpycyOSORnIyk3vupiWF3XQRjFDHBHqScjETWfuZinnu9BFvyWKobWnlrXx21zZ1nTJccH+MLfVJv+HMyEpmckcSE5HhstrC6hJkRZUywA7A4N52WaQ5WrpzX+1y7u5vqhjaq6lt9P9uoamhjZ00Tr+08jqfP8W+H3UZWeoJv6Z5Ejm8pPzkjkay0ROJjzfXDjKExwR4mCQ47M8aPYcb4Mef8ztPt5VhThy/srVTXnw7+5ooGWvuMrSYC45Pje8Oekx5eq/hd3V5aOz00d3hodXto7fTQ0tlNS0fPfevWetZP6343nvZ2XmvYxuSMJHIzEslJT2JyRiJJceajOJzMX3MExNhtZKcnkp2eyHLGnvE7VaWh1U1VQ1ufwFvhX7e3jrp+VvEnZySduW3vC8f4flbxVZVOj/fMkPlCeXYg+wtmS2c3rb7HzZ0e3B7/dhrGxdhwxsXgjI8hyRGDMy6GDKeDwy3w2s4TNLS6z5jeNSaudw2mZ+0l13c/1F9mkcgEO8REhAxnHBnOOBbmpJ3z+za3p3fVvtoX+qr6NnYcbeK1HWet4sfYyE5LoLO9Dd30Rm9APX62wSY67CTFxTAmLoakuBiS4uxMSk3AGWc974yPwemwftcb2rgYnHF2nHGxJMXZcfpeG2vv/4DLm2++ycqVKznV0UV1fRuV9da/p6q+lcr6Nt45cJLfbznzFNmUhFhr6e5byvcNv8sZR5hdcj0gnm4v77V10dDqpr610/rZ4qa+1U1Da+eF3+AsJthhLtERw8zxycwcf+6FUzzdXmoaO6zg+5by1Q1t1NZ1MHlSuhW+uD5BjDszlD2BTYqzlqr2Edyhlxwfy+xJKcyedO6FWPvur6jyhb+6oY2th9/jT9tq6Ps9leiwk5N+euneu4qfkciElIQR/Tf15en20tDmpqHVTYMvoPUtvsC2uvsE13qusb2L/q6PKQKpCbGDnr8JdgSLsdvI8X2I+67iW0vF+SGsbGjOt7/C7fFytLHdCnufJf7+2mbe2FOLu/v0poLDbiM7PaF3CZ/r24TJzUgiKy1hwLWK/vQEtb7F3RvOM4LqC2lPaBvbuvp9HxFIS3SQnuQgI8nBjPFjfPfjyHBaz6cnORjrjCM9yUFaogO7TZBvDO5vaIJtRBRHjI28sUnkjU0653fdXuVYU7sv8Gcu8TcerKe96/ROSpvApLQEK+y+JX5lZRclr+3tXf3tuzrc1D5wUNMTTweyYHxy7/0MpxXY0/cdpPqCGmxBC7aIvAg0+B6WAsXAFUArsF9V/x6seRujk90mZKVZhwwvmXbm71SVupZO3/b86W366vpW/lBew6kOaxAN294DpCU6epeeBROSzwhmum/JmuEL70gFdbCCucR+VVV/2fNARH4L3KKqXhH5jYi8oarmGjrGiBARxo2JZ9yYeJbkpp/z+8Y2N29veIfrrlwZlkEdrGD2iheKyH0i8pCIpACxqtqzAdQAZAdx3oYxKKmJDpIdEhWhBhDtb1fccM5AZAFwNxCnqp/wPfefwEuq+m4/098F3AXgcrkWrV27Nqj1BaqlpQWn0xnqMvplagtMONdWVFRUqqqL/X6Bqgb9BryBFeSexz8Fci70uvz8fA1X69atC3UJAzK1BSacawNKdBCZC8qquIgUiUih734aUAO4RaSnCToNMCMDGkaQBGvnWRnwKRGZDcwEHgQygK+ISAuwWs2OM8MImqBvYw+FiDQDe0NdxwDGAidDXcQATG2BCefaZqjquR07Awj3BpW9OpgdBiNIREpMbYNnaguMiJQMZnozNJJhRCETbMOIQuEe7J+FuoDzMLUFxtQWmEHVFtY7zwzDCEy4L7GNKCMiNhFZLCLzLjy1IQGOIhG2wRaRfxORj4vI10Tk3K79EBGRVBG5X0RuE5FnRSQ/1DX1JSLJIvLXUNfRHxEZD3wZqFPV8lDX00NE7CLyJRH5VxF5TERyQlyPzff5ehm4xPdcnu+8iztE5OYLvUdYBltEZgKZqvocsBa4L8Ql9VUA/E1V1wBP4+trDwe+b/ePAmF35UBfbV8HHlPVqlDXc5Y5WCcpPQv8Grg+lMWoqtf3+doK9CyxHwIeV9VfAFeISOr53iMsgw2swDqHG1XdCywMbTmnqepGVd3ue5iB9ccPFx8E/gKE42VKVgJdwO0i8i0RyQxxPX3tAVaIyCRgAfDnENfTn0xVPeW7vw847xA54RrsscCpPo8HP+hTkIlIItYX0O9CXQuAiMwCGlX1eKhrGcB8oExVnwBeAB4LcT29VLUD+BawBpgChOPF2foO1doEuM43cbgGuxZIhd5VOPf5Jx9ZvpruA76pqoMfQjI4PgBcIiIPAHNE5AHfl0+48PS5v40wOh9fRAqAuapaBPwD+F6IS+pP3wykASfON3G4Bns90NPaNwsYVDvdCLgT+KWqNlxwyhGiqt9T1e+q6neB7b77baGuq4/1nN6kGscFPpgj7Gqs1XGAvxNGXzp9HPcNWAIwnQtsAoZlr7iq7hORGhG5HcgFfhzaik4TkY8AtwDLfUciOlX1M6GtKvyp6lYRqRaRO4AJwFdDXVMfzwP/4dtpmw48EcpifGta1wGLgHgRcWBtKtwnIkeB1/tsb/f/HqZBxTCiT7iuihuGMQQm2IYRhUywDSMKmWAbRhQywTaMKGSCbQyJiKwUkXBs6BjVTLCNIVHVNwmjXn7DYoJtGFEoLDvPjOAQkXuARmAy8DbW6Yl2rDPppgPVqvqkiDiBe4CDQCZQrKrFIuLCOk11HzAV+Lmq1vve+xqsM6M+BFx7oc4oI7hMsEcJEXkfkKCqj/tOYnkGeAq4TH1XRRWRF0TkBeALwG9UdYdv2ldE5P8A3wEeVNVa3ymO44F6rDW/clV9TUS8wMXAayP9bzROM8EePXr6jj+MtZTuGeyg77nbB7GW5vOAhwFUVUWkHUgBJqtqre/5o5y+TJO3z+mitVzglEIj+Mw29uixG2hW1ZdV9ff4gsvpETrAWr0+BGwH8qH3FNUErHOAT4rI9J6JRSR5BOo2AmBOAhklfAF9BOuE/SNY4QX4LLAO6yy63ar6jC+w92CFPBPYpKobfWOB/SewH+sa578FcoBfAZ/HOpXwUaAb+KqqNo/Mv844mwn2KCYiK4HlqvrtUNdiDC+zKj5KiUgS1giYs0RkaqjrMYaXWWIbRhQyS2zDiEIm2IYRhUywDSMKmWAbRhQywTaMKGSCbRhR6P8Dh5ASWh+ZuSUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.1, 10, lang, device, first_train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAC4CAYAAAAooOojAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAF9ZJREFUeJzt3Xt8VPWZx/HPM5lMGDK5M0mQAFJCRMQb8YY3QGu9VFutdFutL3XbLdiuZcV7deu21lbrpeq63aptd1H7YlekitZLbWtB6HppiRbRVgmgUoJMJCEhEwIhybN/nJMwiSEZQk7m9rxfr7ySM+c3Zx5CvnPOnOdcRFUxxmQOX6ILMMaMLAu9MRnGQm9MhrHQG5NhLPTGZBgLvTEZxkJvTIax0BuTYSz0xmQYC70xGcaf6AIORGFhoVZWVia6jH61traSm5ub6DL6ZbUNTTLXBlBTU7NNVcODjUvp0JeVlbF69epEl9GvFStWMHv27ESX0S+rbWiSuTYAEfkwnnG2eW9MhrHQG5NhLPTGZJiU/kxvko92KZ0tnXQ0d/R8dbV2UXxmMQDbnt7Gjtd39MzrbO5EsoXpT053FnAPrH96PWWXlZF3VF4C/yXpy0Jveunc2Ul7pJ3OHb2DG54bJmtUFg0vNNDwTEOveZ3NnVSvrsaX42P9Veupe6Cu90J9MKtjFiLCtl9vI/JIhKyCLPwFfvwFfgJjA3vHhqHuP+vYfN9mco/IpfzSckq/UkpOec7I/iLSmIU+zWmXsnvLbrJLsskKZtH0chPcCmvuWENn895gH7X8KEZXjWbLQ1vYcPWGTyyn8JRCsiZm0fp2Kx//6mP8Bf6e4AamBOhq78KX42PM+WMYdfCovfPznWCjgEDVg1Uc8rNDEJH+C74UTrz7ROr/t56tj2xlw7UbaP1rK1N/MRVVRdsVX459Kj0QFvo0oF2Kdiq+bB9tG9rY8vAW2mrbaFvvfHW1dXHkS0dSdFoRHTs6oBY6DurAn+8nMC6Av8DfE6TizxTj/y9/z1q4O7iBg5y18YTrJjDhugn7rKXotCKKTiva53yff/DAZhdnM+6b4xj3zXG0vtuK+J03iOgbUdZ8eg3hL4Upv6yc/BPy9/3mYfbJQp9iOpo7qF9S7wQ6JthVP62i/LJy9mzfw+b7NhOcHCRYGaTojCKClUGCVUEAxpw3BvKgenZ1v8vPPSyX3MOS5wCU3Kl7a/EFfRR/tpjIoxE+eugjglVByi8tZ9y3xuHPtz/leNlvKslop9K0qql3qGvbKLu0jAnXTqCzrZN189YhASH4qSDBKUGKPl3E6KmjAcibkcepO09FstJvDZg7LZdpv5xGx44OPl76MVsf2cqmuzZRsbACgNa/tpIzIQd/yP6sB2K/nQTYvWU3O9ft7BXsvBl5TLx5IgBvnfkW2q69gp0zztmRFSgLcPz7xzNq/Kh+gy2+9At7X/58P2O/OpaxXx3Lnu17yBqdharyzhffYdeHuwjPdTb/C2cVZsTvY39Z6D2gqrANml5uYmftTtpq2/Dn+3tC/eZJb7Lrg10APcHOneZsxkqWcNQfjiIwLtBvsEWE4MHBkf0HJbHsouyen6seqiLySIT6JfVEHomQMzGHSbdNovyS8gRWmHws9MNEO7UnoGvPWwvPwV/4CwCSLRSdvnfnVuX9lfiCPoJTgv0Gu+CkgpErPE2ICIUnF1J4ciGV91ey7eltbH1kK76As+Nw95bdNDzbQPgfwmQXZg+ytPRmoT8Au7fspuG5Bhqea2DHazuY+eFMfDk+Sr9cSuOkRo743BEEK4OMmtA72GM+NyaBVae/rNFZlF1URtlFZT2PbXt6G7XfrKV2QS1jzh9D+WXlFJ1RFFc3Id1Y6Ieg8feNbLx+I9E3owDkTMghfGGYzmgnvhwf5ZeU827FuxTPLk5wpabbQVccRN4xeUQejRBZHOHjxz8mpyKH49YdR1YwK9HljSgL/SA6dnTQ+NtGGp9rpPxyZ+eQP89PVm4Wk26fRMm5JeQelmv94iQnIuQfm0/+sflMvmcyDc810Pp2a0/gaxfUEpwcpPTiUgLhwCBLS20W+n507upky0+30PBcA80rm9E9ir/QT8GsAgpnFZJ/fD5Hrzo60WWaIfIFfIQvCBO+wLneRFd7Fzte30HdA3VsuHYDxecUU35ZOSXnlvTsE0gnFnqc//TmVc10NHUQvjCML+Bj0+2byA5nU7GwgpJzS8ifmZ+Rn/8ygS/go/r1aqJvR4k8EiHyywgNzzQw+d7JjL9qPNqpaXU+qiehF5E8YAHwPjAbuFNV18fMPxQ4C2gDyoHv4fxarwa2AtXAj1V1kxf1AbTXt9P4QiMNzzbQ+GIjnS2dBA8JEr4wjPiE4949juzizN7Lm2lC00OE7gox6fZJbP/ddvKqnbP8IosjbLp9E3wR5685xXm1pg8DK1V1lYhsAuYD18XMn4/zRrBFRH4CFAAHA9mq+piIvAdcANw/XAWpKq1vtZJ7hPP5e+MNG9m6aCuBgwKUfrmUknNLerXVLPCZy+f3UXJ2Sc909phs/MV+aE9gUcPIk9Cr6kZgozs5CVjZZ8jLwJUicicQUdUmEXkXuF1ExgFHA88faB2drZ1sf2k7Dc86bbX2Le0cs/YYQtNDjL9uPOMWjCN0VMh2wpkBlZxdQsnZJaxYviLRpQwLUVVvFixSDtwI7ARuUdWOPvNvA04HrlHVV9zHTgR+APwJuLnvc9wx84B5AOFwuHrJkiW9B3QCWcBa4BpgDzAaOBY4ATgZCA3Xv3LfotEoodAIvNAQWG1Dk8y1AcyZM6dGVY8ZdKCqevoFTATu7fPY1cAEYBTwEM5n+EOBK9z5n8H5TD/gsquqqrRzT6duf3m7rr9+vb4+7XX94AcfqKrqnh17tHZhrTa+1Kiduzt1pC1fvnzEXzNeVtvQJHNtqqrAao0jk17tyJsB1KvqZqAFOFJEAu6bTDtwkvtGoCKyGJiJs35e4y7id8DXB3ud9jp4JfwKHU0dSLZQcGoBwSnOcen+PD+VP07Oa+Ibk0he7cirAy4RkQactfhNOHvzBbgLeAy41d3JN9F9bBRws4hMBYqBBwd7kfZOJfj5QiZ8oYyiM4rsnGpj4uDVjrwIcI87ucj9/lrM/GXAsj5Pa8Z5Y4jbtkLl1a/lUH3aoDf1MMa4UvqQg1FZsLRmc/d+AmNMHFI69KGA8EHDTmo+3J7oUoxJGSkd+ly/MDqQxdKazYkuxZiUkdKhF4Gzp4/l2bc+oq29M9HlGJMSUjr0AHOrK4ju7uDFd7YmuhRjUkLKh/74ScVUFAVtE9+YOKV86H0+4cIZFfzfhm3UNbUluhxjkl7Khx7gwhkVqMJTb9ja3pjBpEXoJ5SM5vhJxdazNyYOaRF6cHboWc/emMGlTejPOXys9eyNiUPahD43x289e2PikDahB+vZGxOPtAq99eyNGVxahd569sYMLq1CD9azN2YwaRd669kbM7C0Cz1Yz96YgaRl6K1nb8y+pWXorWdvzL6lZejBevbG7Evaht569sb0L21Dbz17Y/qXtqEH69kb05+0Dr317I35pLQOPVjP3pi+ki70IuITkWNE5MjhWJ717I3pzZPQi0ieiNwsIheLyMMiUtln/qEislBErhCR74qIuI+XA9cDH6vqmv6Wvb+sZ29Mb16t6cPASlVdDDwKzO8zfz7wuKo+6I4tcIN/C3Cfqn44nMVYz96YvTwJvapuVNVV7uQkYGWfIS8DV4pIIRBR1SZgNrAHuFxEvi8iZcNVj/XsjdlLvNqr7W6q3wjsBG5R1Y4+828DTgeuUdVXRGQhsF1VF7mf529U1Yv6We48YB5AOByuXrJkSVz1PFXbzjMb9nD3rCAlQe93ZUSjUUKhkOevMxRW29Akc20Ac+bMqVHVYwYdqKqefgETgXv7PHY1MAEYBTwEVAPfAi535wvwx8GWXVVVpfH6cFurTrzhWX3gpXVxP+dALF++fEReZyistqFJ5tpUVYHVGkcmvdqRN0NEKtzJFuBIEQmISMB97CTg76q6C1gMzARWATPc+aVAZDhrsp69MQ6/R8utAy4RkQactfhNwAKcNfhdwGPArSKyCWdL4C5VbRaRTSLyj8BY4NvDXdTc6gquW/oWNR9u55iDi4d78cakhLhC7wbxdSAL+FfgBVVdtK/xqhoB7nEnu8e9FjN/GbCsn+fdHU89Q3XO4WP5t2feYWnNZgu9yVjxbt43q+pfgRuAr+F8Fk851rM3Jv7QF4jICcAmVY0CDR7W5Cnr2ZtMF2/o1wBnAT8SkaNwPnOnJOvZm0wXV+hV9Q1V/a6qNgOtwIPeluUdn0+YW23n2ZvMFVfo3ePjp4nIPwHfAL7jbVnesvPsTSaLd/P+LeBd4CxVvRpY711J3htfPJoTPmU9e5OZ4g39JOABYImIhIA870oaGXOrx9t59iYjxRv6e3EOoFkClAN/9q6kkXH29HI7z95kpHhDr8AsEfkxziG0f/KupJGRm+PnnMOtZ28yT7yhvxbnGPqHgWbgOs8qGkHWszeZKN7Q/11Vn1TVd91DaOu8LGqkHHdwMeOLrWdvMku8oe97EnHucBeSCHZtfJOJ4g39RyKySERuFZH/Bj7ysqiRZD17k2niPSLvOeBfgKeBhZ5WNMKsZ28yzT5PrRWRH+Fc1KJvEgSnbfdrD+saUXOrx3PtE2vsPHuTEQY6n/55Vb2hvxkiMtOjehLi7Onl3PL023aevckI+9y8V9WXB5j3qjflJIb17E0mSbo73CSK9exNprDQu6xnbzKFhd5lPXuTKSz0MaxnbzKBhT6G9exNJrDQ92Hn2Zt0Z6Hvw86zN+nOQt+H9exNurPQ98N69iadeXUDyzwRuVlELhaRh0Wkss/8Q0VkoYhc4V5pV2Lm5YvIb7yoK17WszfpzKs1fRhYqaqLgUeB+X3mzwceV9UH3bEFAG74LwTqPaorLtazN+nMk9Cr6kZVXeVOTgJW9hnyMnCliBQCEVVtch8/D3gB6PKirv1hPXuTrry6VTUiUg7cCOwEbomdp6pPiUg1TsCvccdPA5pUdWvM1n5/y50HzAMIh8OsWLHCk/oBphb7eOyPtRwmmxmopv5Eo1FPazsQVtvQJHNt+0VVPf3Cuf/8vX0euxqYgHP324dw7mF/Pc6bxI1Ajft99EDLrqqqUi89sfrvOvGGZ/XP7zfs93OXL18+/AUNE6ttaJK5NlVVYLXGkUmvduTNEJEKd7IFOFJEAiIScB87Cedim7uAxcBMVb1TVe9Q1TuAte7PO72oL17WszfpyKsdeXXAl0TkcuB7wE3AApxLbgE8BtwqIl8HznCnk4717E068uQzvapGgHvcyUXu99di5i8Dlg3w/Mu9qGso5lZXsLRmMy++s5Xzjx6X6HKMOWB2cM4grGdv0o2FfhDWszfpxkIfB+vZm3RioY+DnWdv0omFPk52nr1JFxb6OFnP3qQLC32crGdv0oWFfj/YefYmHVjo94P17E06sNDvB+vZm3Rgod9P1rM3qc5Cv5+sZ29SnYV+CKxnb1KZhX4IrGdvUpmFfgisZ29SmYV+iKxnb1KVhX6IrGdvUpWFfoisZ29SlYX+AFjP3qQiC/0BsJ69SUUW+gNkPXuTaiz0B8h69ibVWOgPkPXsTaqx0A8D69mbVGKhHwbWszepxEI/DKxnb1KJZ7eqzjQXzqjgvt/X8tQbm7nytCmJLqfHjl17qI1EWV/fwrpIlNr6KOvrdhKsWdEzJvY23H1vyB17h26JmTvQnbv3tbxey+rz/O5lt7S0kbf2jyhOC7S7E9rzne7pT7ZI947RAZ/T88xB5se+hKLsatvF4kNbmFKW94nXTiWehF5E8nBuWPk+MBu4U1XXx8w/FDgLaAPKcW5yWQB8DYgAZwLfV9V1XtTnhdie/T/PqRzx12/euYfa+hZq66Osi7Swvj5KbSTK1h27esbk+H1UloYYG/JRXpbvPNjnDztWrz/6fYzrm73Yyd7z4nuOtEcZk5fj/Nz9WM8bhPSa/uT8vW8ePWOk9+N84rnS77J6v1k5U5FIhGAgi1Tn1Zo+DKxU1VUisgmYD1wXM38+zhvBFhH5CU7gDwV+q6prRaQOmAdc61F9nphbPZ5rn1jjac++aWe7u8ZuoTbme33L7p4xwewsKktDnDi5hClleUwpDTGlLERF0WiyfMKKFSuYPXuGZzUeCKe2YxNdRr9WrFhBRdHoRJdxwLy6a+1GYKM7OQlY2WfIy8CVInInEFHVJuDVmPklwF+8qM1LZ08v55an32ZpzWbOKjmwZTW2trMu4qy510f2bppvi+4N9+hAFlNKQ5wyJcyUshBVZSGmlOYxrjCIzzfA9rfJaOLV4aMiUg7cCOwEblHVjj7zbwNOB65R1VdiHh8N3A5cr6q76UNE5uFsBRAOh6uXLFniSf1D9fO1u1m9tYMfHqcUF4QGHKuqtLRDXbSLumgXW1q72OL+3NK+d9yoLDgo5GNcyOd+Fw4K+SgeJfgG+nC9D9FolFBo4NoSxWobujlz5tSo6jGDjfMs9D0vIDIRuEpVF8Y8djWwFKgH7gceVtUacT483QT8VFUbB1v2IYccou+9955HlQ/Naxsb+PLDrzHviBxuuvjTgBPuj6O7WR+J9qy9a+uj1EZa2L5zT89z83L8VJaFqCrNY0pZiMrSEFVleYwtGNVr59iBcjahZw/b8oaT1TZ0IhJX6L3akTcDqFfVzUALcKSIBABUtR04CbhXVVVEFgMzgRqcHXmL4gl8suru2T+/cRctT65lvbtzrSk23KP8VJXlcdb0cipLnc/cVWV5lOXnDGu4jemPVzvy6oBLRKQBqMZZey/A2Sl6F/AYcKu7k28icJeIfAG4CDjZ/cPfrarzParPMz6fcPFxE/nRb97l+bUfUVUW4uzpY3s+b1eVhQjnWbhN4ni1Iy8C3ONOLnK/vxYzfxmwrM/TnnS/Ut78Uz/F+PZNfPaM2RZuk3TsiDwP+HxCKCAWeJOULPTGZBgLvTEZxkJvTIax0BuTYTw/OMdLItICJNfROXuNAbYluoh9sNqGJplrAzhEVQc9BTDVT619L54jkBJBRFZbbfvPahs6EVkdzzjbvDcmw1jojckwqR76hxNdwACstqGx2oYurvpSekeeMWb/pfqaPqmJHYdrklDKhl5ErhSRr4jId0SkONH1dBMRn4hcIiLLgBMTXU8sESkUkWvc+h4TkapE19RNRPJE5GYRuVhEHhaRkb/Q4CBEJF9EfpPoOvoSkSdF5Ofu1zcGG5+SLTsRmQqUqep/iMghwELgOwkuCwBV7QJ+6f7RJtuaPpmvQzjYdRUTyt1quxDnwi/J5hlVXRTv4JQMPXAKzkU3UNX33It2mEGoatJehzCO6yom2nnAC8CsRBfSj8NEZCGQCzygqs0DDU7VzfsxwI6Y6exEFZKK3OsQngI8kehaYolIuYjch7NF8kKi6+kmItOAJlVNyvuWqep1qnov8Bxw92DjUzX09UAh9Gx2tQ883HRzf18Lge/1d+HRRFLVrap6FfAQzhWWksW5wIkiciNwuIjc6L5xJhVVfRNnK2lAqbp5vwq4HOdKO9OAuA4/NECSXoewv+sqJrikHqp6Z/fPIjJVVe9IZD2xRGQOzu/tHREpAgbdGknJ0KvqOhHZIiKXAwfjXFE3KbhrgM/iXBtwlIgEVPUPCS4LgCS/DmF/11U0g3sT+KqITAemEsfvzQ7OMSbDpOpnemPMEFnojckwFnpjMoyF3pgMY6E3JsNY6I3nRGS2e1tykwQs9MZzqroCsPMjkoSF3pgMk5JH5BnviMgCoAnnbsIrgQuALJyzGqcAm1T1IREJ4dyJeANQBryuqq+LSBjnlN11wGTgZ6ra4C77TOBo4HPAWaq6AzPiLPSmh4icBgRV9d/dE3MeBX4BnNp9vraILBWRpcDVwP+o6tvu2KdF5PPA7cBNqlovIuOAcqABZ6tyjaq+KCJdwEzgxZH+NxoLvemt+3yB83HW7h+6j3fFjNmAsxVwJPBdAFVVEWkDCoCJqlrvPl6Hc0w9QFfMqan1OBfNMAlgn+lNrL8BLaq6TFV/hRtqel8BaDLOxS7WAlXQc7puEGgGtonIlO7BIpI/AnWb/WAn3JgebnhvBQLAZpxgA1wBLMc5o/FvqvqoG+YFOG8AZcBrqvqqiEwAfgjUAo3A48AE4BHgWzhX6/kB0Al8W1VbRuZfZ7pZ6M2ARGQ2cLKq3pboWszwsM17s08ikotzRd9pIjI50fWY4WFremMyjK3pjckwFnpjMoyF3pgMY6E3JsNY6I3JMBZ6YzLM/wNiCIFwmSgu1gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.00001, 5, lang, device, first_train=False)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 1.572, PPL 4.848, 4.0 min on cuda:0\n",
      "loss 3.388, PPL 30.146 \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPcAAAC4CAYAAAA/gnrqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAGXJJREFUeJzt3Xl8VPW9//HXZ7KTPSFsYQ8JIeyCoiAKapVWEaTen9fl3tpyxWpb1KKWRRGxbnj90Va9ba1tuW63Upu6s+gVVBBUcAsRCFtkB1kCBMhC5nP/mEnIniFkcmYmn+fjkQeZ+Z45886Qd+bMmTPnK6qKMSb0uJwOYIzxDyu3MSHKym1MiLJyGxOirNzGhCgrtzEhysptTIgK98dKRSQemApsA8YA81R1c7XxMcBtwDHvVU+p6lf+yGJMWyX+OIhFRHoD6ar6kYhcCExQ1XuqjY8BClW1sMXv3BgD+OmZW1W3Alu9F3sBH9ZapBS4QUTAU/KX/ZHDmLbML8/cACLSCZgOnABmq+qpBpabD+Sq6kf1jE0BpgBER0cP6969u1+yni23243LFZi7Lyxb8wRytoKCggOqmtbkgqrq1y+gBzC/kfErgHuaWk9WVpYGqmXLljkdoUGWrXkCORuwRn3onl/+NInIOSLS1XvxGDBYRCJFJNI7/jPxbpMDGUC+P3IY05b55TU3sAu4SUQOAsOAmXj2ngvwBLAdmCQiCUAssMhPOYxps/y1Q20f8KT34gLvv6urjb955us8+1zGtCX+euZucboFVvVYhSva5fmKctHl1i50ntyZ8qJyCqYUVF1fuUzq1akkjU6ivKicfS/uO31b73JxQ+KI7hZNxYkKTm45WeO2rmgXrnYuXOGBuVPF+EZVcZ904y5zo2WKlivuMjdh8WFEto/EfcrNsc+OoWVatYy7zA0RTic/e0FT7uNRStnwGNIiI3GXuHGXuJEoz8t2d4mb4+uOV13vLnHjLnUT1S2KpNFJlO0uY/MvNtdZZ9/n+tJ5cmeOrzvO5yM+rzOe87ccOlzXgaIPilg3cV3N4ke7yHw6k8RRifAN5P0mr85416ldiekdw/H84xxacqjOePKlyYQnhlO2r4zSnaW4ol1IlFSNRyRHIGFSJ5eT1O395a+AsNgwAEr3llJxrKJGQSRKiB8SD8Dh5YcpP1BeYzyiQwQkeda58+mdlO8rr1Gu2P6xpN+eDsCGyRso/67mePKlyfR6sBcAnw38jFNHT51ef7nS6cedyPxtJrjho9g6b8TQ7Z5uZMzLwH3czRcjv6j7g/7JDw9eKwuach9JVu4fcZgld15ETGRYjbGoTlGc9815Dd62Xd92jNw/sqr0lX8AortHAxCTEUP/V/vX+eMQNyQOgIiOEXT8t441busuceNq531WPwklhSVV12up4i5x0/GGjsT0juHop0fZMm1LnVzD84YTlxjH/oX72Ty17h+fEdtGENMzhu3ztlM4t7DOlsnQFUOJSI5gz5/38N0/vquxVeKKdtHnqT4A7PuffRQtL6rxy48L+v+tPwCFcws5tOhQjfKEJ4Uz7JNhAORfl8/BNw7iLveUGiAmM4YRBSMA+OZfv+HIB0dqZI8bGsfwz4cDsOXuLRSvLa4xnnhRIjzo+X7X07s4WeDZcpJIQSKElHEpVeUu2VbCqaJTSKTgijz981eKHxEPFSARUrVMwsgEACRM6P14byTCc33leOyAWABcsS4GLR5Udb+Vy6zdu7bB36dgETTlbh/tYvuhE/zu/U38alz2Gd1WwoTItMgGxyNSI0j7YcNvG8Zmx5L5u8yG72AYnDvt3AaHO/5bR9ImpdX54xLTJwaA1PGpRPeMrvnHpcRNRHvPtmH8sHjSb0uvM+6K8vyCu0vclB8orzOe+Ywnc/EXxRx842CNX/7KZ10AiRTCEsKIiIyoGg9POf2rkXJFCtE9oqvGJEKqsgH0mNmD8lvKT49HCuHJp2+f81IO7nJ3jXK5Ylys/GolAOfln9foFsqQ94c0/NgD2c81/vvQ/d6Gj49whbtIuSKl7sDyRlcZFIKm3NHhMGF4V579cCtXD+5Cv84JTkfymSvchSvRBYn1j8f0jCGmZ0yDt0++NJnkS5MbHE//WTrpP0tvcDxjXgYZ8zIaHO8xvQc9pvdocLzzTzo3OAaQcnk95aimXd92jY4H2kuPUBFUe4tm/qAfSTERTM/No8Jtu8+NaUxQlTupXSSzx+fw1Y4iXlhV6HQcYwJaUJUb4OrBXbg4K40nlmxkd9FJp+MYE7CCrtwiwq8nDsCtMPv1dZXHpxtjagm6cgN0S2nHL7+XxXvr97N43V6n4xgTkIKy3AA/HtWT/l0SeOCNfI6cLHc6jjEBJ2jLHR7m4rFJgzhQXMq8xRucjmNMwAnacgMM7JrIT0b14qVPtvNZ4SGn4xgTUIK63AB3fS+L9KQYZuTmUXqqwuk4xgSMoC93bFQ4v544gM37i/njB1ubvoExbUTQlxtgbHYHxg/uwtPvb2bLd8VN38CYNiAkyg0w+6ocoiNczMjNw22HphoTOuVOi49i1pX9+HTbIf6+dofTcYxxXMiUG+D/De/GiF4pPPz2er47Vup0HGMcFVLlFhEemTSQknI3c9/6xuk4xjgqpMoNkJEWx88v6cObX+1m2Yb9TscxxjEhV26An16cQWaHOO57bR3HS+ud6MSYkBeS5Y4Md/HopIHsKjrJ/HcLnI5jjCNCstwAw3umcOOI7vxl5Tbydh5p+gbGhJiQLTfAveOyaR8XxfTcrzlV4XY6jjGtyl9zhcWLyCwRuUFEnhWRPrXGU0XkARG5UURu90cGgMSYCB68uj/5u4/y15WF/robYwKSv56504AP1TPv9vPArbXGpwEvq+pLQLqINHLe4LMzbkAnLuvXkf//bgE7Dp3w190YE3D8Um5V3aqn59vuBXxYa5FBqrrJ+/2XwCh/5ADPe99zJ/THJXDfa3ZaJtN2iL9+2UWkEzAdOAHMVtVT1caWqurl3u8vBwar6hP1rGMKMAUgLS1t2MKFC5ud591vy3lpfRk/HRTF+V1a9nTtxcXFxMXFteg6W4pla55AzjZ27Ni1qjq8yQV9mcT7bL6AHsD8Wte9Ve3764B/b2o9WVlZZzVh+akKt1799Ao9Z+5SPXy89KzWVVsgT9Ru2ZonkLMBa9SH7vlrh9o5ItLVe/EYMFhEIkWkck6fL0Skcg6YocAKf+SoLswlPDZpIEdOlvPIO+v9fXfGOM5fO9R2AdeJyM14pnubCUwF7vCOzwduEJEfAdtVtVXOstCvcwK3XNSbhWt28vGWA61xl8Y4xi9zhanqPuBJ78UF3n9XVxs/BMz2x3035Y5LM3knbw+z/rmORXeMJjoirOkbGROEQvoglvpER4Tx8MSBbDtwnGeW1Z0215hQ0ebKDXBhZnsmnZPO75dvYePeY07HMcYv2mS5Ae67Mof46HBm5H5tp2UyIanNljslNpL7r8rh8+1FvPTpdqfjGNPi2my5Aa4Zms6Ffdozb9EG9h4pcTqOMS2qTZdbRHj4mgGUVbiZ80a+03GMaVFtutwAPVJjufOyLBbn72VJvs0YakJHmy83wH+M7kV2p3geeD2fYyU2Y6gJDVZuICLMxWM/HMS+YyX855KNTscxpkVYub2GdEviRxf05PnV3/L59sNOxzHmrFm5q7n7ir50Sohmxj/yKDtlp2Uywc3KXU1cVDgPTRjAxn3H+NNHNmOoCW5W7louy+nIDwZ24rf/u4ltB447HceYZrNy12PO+P5EhbuYmZtnp2UyQcvKXY8OCdFM/342q7Ye5NW1O52OY0yzWLkbcP253RneI5mH31nPgWKbMdQEHyt3A1wu4dFJAzleeopf24yhJghZuRuR2TGe28b04bUvd/NBwXdOxzHmjFi5m3D7mAx6p8Uy6595nCizGUNN8LByNyE6IoxHrhnIzsMn+e17m5q+gTEBwsrtg/N7p/Kv53bjuRXbWLfLZgw1wcHK7aMZ3+9HcrtIZuTmUWGnZTJBwMrto8R2ETwwPoe8XUdY8HGh03GMaZJP5RaRH4tIjogMFJFXvJMNtDlXDerM2L5pPLl0IzsP24yhJrD5+sx9RFW/AX4FTAai/RcpcIkID00cgCrMfj3fDk01Ac3XcieKyPl4pv4pBg76MVNA65rcjmmXZ/H+hv28nbfH6TjGNMjXcn8FjAMeF5EhQOfGFhaRJBGZJiI3icgLIpJVa3yMd/P+Oe/X4ObFd8bNI3syMD2ROW98w5ETdlomE5h8mitMVT8HPgcQkePAH5q4ST9gqarmicguPHNs311rmV+pauGZxQ0M4WEuHp00kAnPrOSxxeu5IsXpRMbU5VO5RWQOsBAYCeQAx4H7G1peVVdVu5gKfFlrkVI8s3wCFKrqy75HDgwD0hOZfGEvnv1wKz3Pi2aM04GMqcXXWT6/BjYAc1X1Wu/Uu00SkXbAaODe6td7y7/Ku8x8Edmhqh/Vc/speJ71SUtLY/ny5T7GbR3DIpX2McJf8k7SO2kZES5xOlIdxcXFAfe4VbJs/uVruXsBTwELRSQOiG/qBuJ5Wr4LeFBVG/vM5GLgfKBOuVX1WeBZgL59++qYMWN8jNt6orrt5+a/fsa6inTuuiSr6Ru0suXLlxOIjxtYNn/zdYfafOAJVV0IdAI+8+E2k4EF3rm4EZFIEYn0fv8zb/kBMoCgne5jTN8OnN85jP9avpnN+23GUBM4fH3mVuBiEZmKZ8/5840tLCKTgOuBC70dLgU2AQI8AWwHJolIAhALLGpW+gBxQ3YU64vKmZGbxytTLsAVgJvnpu3xtdx3A1vwbCJnA/cA8xpaWFVzgdxGxt88g4wBLyFKmHVlP+599Wv+9tkObhjR3elIxvi8Wb5DVXNVdYOqvgbs8meoYPQvw7pyfu8UHl20nv1HbcZQ4zxfyx1X63JsSwcJdiLCI9cMpPSUmwfftNMyGef5Wu49IrJAROaKyF8BO+6yHr3T4ph6SR/eztvDe9/sczqOaeN8Kreqvg3cAbyO5+0t04ApF2WQ1TGO2a+vo7jUTstknNPgDjUReRw4B8+e8hpDeN4OC6mdYi0lMtzFo5MGce0fPubJpRt5YHx/pyOZNqqxveXvqOqv6hsQkQv8lCckDOuRzE0jerDg40ImDElnSLckpyOZNqjBzXJV/aCRsVUNjRmPe8b1pUN8FDNy8yivsBlDTeuz0yz5SUJ0BA9ePYD1e47y5xXbnI5j2iArtx+NG9CJy3M68pv3Cvj2oM0YalqXldvPHpzQn3CXi/teW2enZTKtysrtZ50TY7h3XF8+2nSA1760A/tM67Fyt4IbR/RgaPckHnprPYeOlzkdx7QRVu5WEOadMfToyXIefnu903FMG2HlbiXZnRK49eLe/OPznazcfMDpOKYNsHK3ol9ckknP1HbM/GceJeUVTscxIc7K3YoqZwz99uAJfve/NmOo8S8rdysb2ac91w7ryrMfbmX9nqNOxzEhzMrtgFk/6EdCTITNGGr8ysrtgOTYSGZflcOXO4p4cfW3TscxIcrK7ZAJQ7owOrM98xZvYHfRSafjmBBk5XaIiPDwxIFUqNqMocYvrNwO6p7ajrsuy+K99ftYkr/X6TgmxFi5HTb5wl7kdE5g9uv5HC2xGUNNy7FyO6xyxtADxaXMW7zB6TgmhFi5A8DgbkncPLIXL67ezprCQ07HMSHCyh0gpl2eRXpSDDNy8yg7ZadlMmfPL+UWkSQRmSYiN4nICyKSVWs8VUQeEJEbReR2f2QINrFR4Tw0sT+b9hfzxw+2OB3HhAB/PXP3A5aq6ovAX/DOsV3NNOBlVX0JSBeRTD/lCCqXZHfkykGdeer9zWz5rtjpOCbI+aXcqrpKVfO8F1OBL2stMkhVKz858SUwyh85gtED43OIinAxMzfP3vs2Z8XXWT6bRUTaAaOBe2sNRVb7/gjQs4HbT8H7rJ+Wlsby5ctbPmQLKC4ubtFsP8xwsSD/EA+99B4XdY04q3W1dLaWZNn8TFX98oVnZpJZQEo9Y29V+/464N+bWl9WVpYGqmXLlrXo+ioq3Povv/9YB81ZovuPlpzVulo6W0uybM0DrFEfOujPveWTgQWqeghARCJFpPIZ+wsRyfZ+PxRY4cccQcflEh6ZNICTZRU89JbNGGqaxy+b5SIyCbgeuFBEAEqBTXiezZ8A5gN3isgWYLuqbvVHjmDWp0M8t4/N4DfvbeKac9IZ27eD05FMkPFLuVU1F8htZPwQMNsf9x1KbhuTwZtf7ea+f67j3V9eRLtIv+4iMSHGDmIJYFHhYTw6aRC7ik4y/90Cp+OYIGPlDnDn9Urh+vO68+cV21i364jTcUwQsXIHgenfzyY1LorpuV9zymYMNT6ycgeBxJgI5ozvz7pdR1nwcaHTcUyQsHIHiR8M7MSl2R14cmkBOw6dcDqOCQJW7iAhIsydOAARbMZQ4xMrdxBJT4rh7sv78kHBd7z59R6n45gAZ+UOMj8a2ZPBXROZ+2Y+RSdsxlDTMCt3kPHMGDqIwyfKefQdOy2TaZiVOwjldEngltG9eWXNDlZtOeh0HBOgrNxB6o5LM+me0o5ZNmOoaYCVO0jFRIbx8DUD2HrgOP+1bLPTcUwAsnIHsdGZaUwams7vP9hCwb5jTscxAcbKHeRmXdmPuKhwZuTm4bYZQ001Vu4glxoXxX1X5rD228O8/Ol2p+OYAGLlDgGTzklnVJ9UHl+0gX1HS5yOYwKElTsEVM4YWlbhZs4b+U7HMQHCyh0ieraP5Y7LMlm0bi9LbcZQg5U7pNwyujfZneKZ/Xo+x2zG0DbPyh1CIrwzhu47VsKTS+20TG2dlTvEDO2ezI8u6Ml/ryrki+2HnY5jHGTlDkF3X9GXTgnRzMjN45S9991m2blyQ1BcVDhzJwzglufX8JyEsYFNVWPe88jXUf1qQepcL2ewbN11V1um2vVbCsvZtnJbPeuXeu6n/gBnmsuX5RHYvPsUY2r/IEHGyh2ivpfTkeuGd+OVNTtYvSeAX39vCMwZVWLCYabTIc5Sq5RbRETtvECt7vFrB/G9lINcfPHFVH/0ldMXGvpf8WV5rbG8NnB99Qs117lixUpGjRpV37BP61PqD+PL8k39HJ+sXk2w81u5RcQF3ABci2cKoZW1xnsCzwCV5wt6WVXf91eetircJUSEBeaulbhIITk2sukFHbA5JjAfszPht3Krqht4UUT6UOslUzVPqOpyf2Uwpi1z8jV3BfB9ETkXOA78wfsHwRjTAsTfL4VFZA7wnqo2OE2viNwJHFTVF2pdPwWYApCWljZs4cKF/ozabMXFxcTFxTkdo16WrXkCOdvYsWPXqurwJhf0ZRLvs/kC5gAXNrFMX+CZxpbJyso660nL/SWQJ2q3bM0TyNmANepD91p1s9y7ky1KVU+KyH8AL6hqKZABNPpxpoKCgmIR2dgaOZuhPXDA6RANsGzNE8jZ+vqykD/3lrcDrgSGAdEiEgmkAhcDPwe+Bq4XkVN4yv1oE6vcqL5sijhARNZYtjNn2ZpHRNb4spw/95afAP7u/aru797xT4FP/XX/xrR1wf9mnjGmXsFU7medDtAIy9Y8lq15fMrm97fCjDHOCKZnbmPMGbBytxBp6LOUxjgkKMotIj8XkRtF5H4RSXE6TyURcYnITSLyGjDS6TzViUiSiEzz5ntBRLKczlRJROJFZJaI3CAiz3o/fxBQRCRBRBY7naM2EckVkee8X7c1tmzAf55bRLKBjqr6tIj0Be4C7nc4FuDzh2Oc0g9Yqqp5IrILz2G8dzucqVIa8KGqfiQi24FbgXsczlTFuxX2Q2C/01nq8YaqLvBlwYAvNzAaWAugqhtF5ByH8wQFVV1V7WIq8KVTWWpT1a3AVu/FXsCHDsapz3hgEZ4DrgJNfxG5C4gFnlLVIw0tGAyb5e2Bo9UuRzgVJBh5jxQcTd2DiRwlIp1E5Dd4tjAWOZ2nkojkAEWqGpAnf1fVe1R1PvA28J+NLRsM5d4PJEHV5lKZs3GCh/fxugt40HsMf8BQ1b2qeifwRzwn8wgUVwEjRWQ6MFBEpnv/QAYUVf0Cz1ZPg4Jhs/wj4GYgF8gBfDqu1gAwGVigqoecDlKd96XVflXdCRwDBjscqYqqzqv8XkSyVfUxJ/NUJyJj8Txu+SKSDDS6dRHw5VbVAhHZLSI3Az2B3zqb6LT6PhyjAXKqKBGZBFwPXOh9l65UVW91NlWVXcBNInIQz2MX7OcibC1fAD8RkQFANk08bnaEmjEhKhhecxtjmsHKbUyIsnIbE6Ks3MaEKCu3aTEiMkZE5jW9pGkNVm7TYtQzwYQdHhwgrNzGhKiAP4jF+IeITAWKgB54PrhxDRCG50M6mcB2Vf2jiMQBU4EtQEfgE1X9RETS8HzSrADP2Wv/pKoHveu+AhgKXA2MU9WjmFZn5W6DROQSIEZVf+c9/vx54M/ARZUfJxSRV0XkVeCXwP+o6jrvsq+LyAQ8p6Keqar7RSQd6AQcxLM1+JWqLhERN3ABsKS1f0Zj5W6rKg+XnYjn2fpb7/XV52rbgudZfTCeWWNQVRWRk0Ai0ENV93uv34XnkFIAd7VPVO3H89lt4wB7zd02rQeOqeprqvoPvOWl5gknMvB85joPyIKqT5nFAEeAAyKSWbmwiCS0Qm5zBuzY8jbIW9K5QCSwE0+BAX4KLMPzAZ31qvq8t7RT8RS9I7BaVVeJSHfgEWATcAh4BegO/DfwCzwnh3gYz2yuM1T1WOv8dKaSldsAnveo8UzY+Guns5iWYZvlBhGJxXOCxxwRyXA6j2kZ9sxtTIiyZ25jQpSV25gQZeU2JkRZuY0JUVZuY0KUlduYEPV/SUlc2m0WG5oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPQAAAC4CAYAAADUtcHpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAGXRJREFUeJzt3Xl4VIW5x/Hvm5kkkD1ACGvYCYEEFFAropK64ApCa6uVe7V1vWq11g1wKS6ItbV2ub22XO11aeu9tA2IWhUVUrVQFUQNiEGWsAlE9kwSQpJ57x8zCWFLhpCZc2byfp5nHpKz/hLynnPmzDnvEVXFGBMb4pwOYIxpO1bQxsQQK2hjYogVtDExxAramBhiBW1MDLGCNiaGWEEbE0OsoI2JIVbQxsQQr9MBmpORkaEDBw50OsZRVVZWkpyc7HSMo7JsrePmbMuWLduhqlktTefqgs7Ozmbp0qVOxziq4uJixo0b53SMo7JsrePmbCKyIZTp7JDbmBhiBW1MDLGCNiaGuPo9tK6DpScvxZvpxZvpJb5TPL3v6k1SbhLV66up+LDikHHeTC/eDC8SJ05HN8YRri7oqnhFusXjr6in6osq6nbX0f2G7gDsKd5D6Q9Kj5hn9CejSRmRwvY/bWfTk5sOLfZMLzn35hDfKZ6q1VXs37D/kHHedNsYxCr/AT8SL4gINVtrqNlUQ31FPfW+wEvrFXKcTnniwlLQIpIK3AasB8YBTwBpwDlAJfClqr7V0nJ2pCsvXO3nV1eMPGJc1rezSDs1jdrdtdTtrmt8JeYkAuBJ9ZDQI4G63XVUrqykbncdtbtq6X1nbwC2v7idDY8eduJQYOyesXjTvGz+9WZ2vrLzYLFneonPjKf3Pb0REdgEFR8fPELwptnGoK346/yBYmtScPW+elJHp+JN8+Ir8bF7wW7qffXUVdQ1jh/45EASshPY9vw2Nj6x8eC8FfVorTKmfAwJWQls+e0WNs7ceMg6PSkeeMWhH7gNhWsPnQW8q6rvichG4EYC278rVdUvIi+JyEJVrW9uIemJwsuffMWkk3syLrfrocFTvXiHHTt+lwld6DKhyyHDmrZb6nFTDzLPz2zcENTurqVuVx2eVE9wYqj31bN/0/7GaSRByLk3uBl/Hpa9s+zgwuOgY/+OnPblaQCUPVJG5YrKxg2BN9NLYk4i2VdkA1C9rhriID4zHk+aJ7CRiEL+On9j4XjTvXhTvdTuqmXv+3sPKcZ6Xz1Z3w58jFqxvIKyh8oOHV9RT96f8sgYm8HXf/2aVVeuOmJdI/81krTT0qj4sIK1d60FIC4pDk+KB0+Kh7p9dSRkJ+DN9JI8NLlxuCfFgyfVQ1xi4JRR9lXZpI9Jx5PqOWSaJaVLIveLC5OwFLSqrgPWBb/tB7wPXK2q/uCwXUBvoKy55aQnCr2ykrl/3goW3HEWSQknFrdp0ST2TCSxZ+Ixp+11ey963d6r8XtVxV/tPzjB92DYrcMCxb4rsEEQz8HlH9h2AN9yX+PGgnpIzk9uLOhVU1axb8m+wMRx4M30knF2Bvl/ywdg3bR11O2tO+QtQ8dBHck4MwOAmq01jX+IoW4MVBURQeuVqi+rjiio5KHJpIxIoXZPLRtnbTxiD9n9hu50/XZXqkqrWD52OfW+evz7D/5Ocp/NpfsPulP9ZTUrJq44Yv1JQ5KgK/hr/Oxftx9PqgdvupfEnol4UgJfA6SOTGXAUwMCw1K9jT9nUl4SAF2v6krWd7LwJHkO+Z03ONrGvKnkvGSS845yAcmR7+CijoSrSaCIdAOmAlXA08BMVf334LjHgLmq+tFR5rsBuAEgKytr1CO/e4lZH+7ngr7xXDEkISxZW8Pn85GSkhLaxApUA/uBTsFhy4BtgA+oCL6ygCnB8XcQ2CT6gIaaGQPMDH49GdgNeIDU4Ots4FrwVfhImZYSWGc1gf+BamACcEvw64uOkvPfgB8Ae4DvAh2BpOC/HYFJBN407QGeazK8YZoCAsdh1cDGJuMbXp7j/L1FmJuzFRYWLlPV0S1NF7aTYqq6DfiRiPQBfkzgT65BJrD9GPPNBmYD5Obm6o2Tz6GMz5izdDO3Xnoa+T3TwxX5uJzwVUUtzbo88I+qUl9RT93uOhDokNMBgK2/2Ert17WHvGVIPyWdXuN6UVxcTKe+nYiLj2s83PSkeEg/K50u47qgqpT/ufyIQ86E7ATiM+MDK65pId9lrfux3Xw1lpuzhSpcJ8VGAuWqupnAvmcE8LWIeILvmzOBLaEub+oFebz1eTnTikqYd8sZeNrRyScRwZsWOOnWVPdrujc73/BXhze7zOwrs9skn3GXcO2htwBTRGQnMAqYTmCbf6+I+IBnWjoh1lR6UjwzJgzl1j8v57nFZVw7tl94UhsT5cJ1Umw78GTw2+eajFre2mVeXNCdoiFbeHJBKeOHZdMrM+lEIhoTk6Lm0k8R4eGJwwB48OWV2BM/jDlS1BQ0QK/MJO48P5eFX5TzWslWp+MY4zpRVdAA14zpy/Be6cyY/zl7q2qdjmOMq0RdQXvihMcmFbC76gCPv3Hk1UTGtGdRV9AA+T3TuW5sP176cBMfrNvpdBxjXCMqCxrg9nMH0btTR6bNLaGmLuRPwIyJaVFb0EkJXh69rIB1X1fyX4vWOh3HGFeI2oIGOHtwFhNP6sF/Fa9hTXmF03GMcVxUFzTAA5cMJSnBy7SiEvx++2zatG9RX9BdUhK57+I8Pirbzf9+tMnpOMY4KuoLGuDyUb34Rv9OzHp9FeX79jsdxxjHxERBiwQ+m66p8/PQq587HccYx8REQQP0z0rhtm8O5LXPtvLOqqPeam1MzIuZgga44awBDM5O4YF5K6isqXM6jjERF1MFneCNY9bk4Wzdt58nF6x2Oo4xEReujiUZwLUE2gyNBx4h0NFqSJPJblXVNj+DNapPJlNO68Nzi9cz8aQejOid0darMMa1wrWHzgMWqOofgT8QaPq3Q1Wva/IK2+nouy/IJSs1kalFJdTW+1uewZgYEZaCVtUlqloS/LYz8AngFZFbROQRETklHOttkNYhnocm5LNq6z7+8P76cK7KGFcJ66NwRCQJOBO4R1VrgsMSgFdE5HJV3ReudV+Q343zh2bz1NuruTC/OzmdrWWRiX3h7MstBJoDPq2quw4bNwsoCqUv95w5c1qdYdd+P9Pfq2Zghoc7Rye26dMp3NzD2bK1jpuzhdqXG1UNywu4DugZ/LoDcF2TcS8AnVtaxuDBg/VEPffP9drn3ld17sebT3hZTS1atKhNl9eWLFvruDkbsFRDqLtwneWeDFwJjA3uFWuA5SIykcCzFV5W1Yh0JpjyjT7MXb6Fh1/9nLMHZ5GZ7J6nbxjT1sLVxrcIKArHso+XJ06YNbmAS3/zPjP/voqfXz7C6UjGhE1MXVhyLHnd07jhrP78ddlmFq/Z4XQcY8KmXRQ0wG3nDKJP5ySmzy1hf621LDKxqd0UdId4D49NKqBsZxX/uXCN03GMCYt2U9AAZwzswrdG9uJ3/1hL6TZrWWRiT7sqaID7Ls4jrWM8U4s+s5ZFJua0u4LulJzAA5fksXzjHv70wQan4xjTptpdQQNcdlJPzhzUhZ++Ucq2vdayyMSOdlnQIsLMywqo8/v5yfwVTscxps20y4IGyOmcxI/OHcybK7fzxoptTscxpk2024IGuHZsP/K6p/GT+Suo2G9PsjTRr10XdLwnjscnF1BeUcPP3ix1Oo4xJ6xdFzTAiN4ZXH16X1781waWbdjtdBxjTki7L2iAu8bn0i2tA9OtZZGJclbQQEqil0cm5lO6vYLZ765zOo4xrWYFHXTu0GwuKujGr975kvU7Kp2OY0yrWEE3MePSYSR647hvbklDZxVjokpYClpEMkTkThGZIiIvishgERkpIneLyM0icl441nuiuqZ1YOqFQ1i8did/+3iL03GMOW7h6vrZ0Je7RES2EGj61xu4UlX9IvKSiCxUVdfdmHzlKTnM/XgLj772OYW5WXROSXQ6kjEhi1Rf7hIgXlUbTiHvIlDgrhMXbFlUWVPHo6+tcjqOMcclUn25fwGc02TUXiALKDvKPE3b+FJcXBzOiMd0UV8vc5dvYYBnB/ldjvw1+Xw+x7K1xLK1jpuzhSyU1qCteQEC3Ad0AhKAuU3GPQ3ktLSMtmjj21rVB+q08OeLdOxP39Gqmrojxru55atlax03ZyPENr7hPMt9LfCcqu5S1QPAARHxBMdlAq4+69TQsmjTrmp++Y49ydJEh0j25X4cuFdEfMAz6sITYof7Rv/OXHFKb555bz0TRvRgWI90pyMZ06xI9+VeHo71hdO0C/N4e1U504pKmHvzGXji2u5xOsa0tVYdcovIqW0dxK3Sk+J58NKhfLZ5L88vLnM6jjHNOuYeWkRuBiYBh18yJUA3oCCMuVzl0uHdKfp4Mz9fUMr4/G70zOjodCRjjqq5PbRPVc9T1fMPe50HPBipgG4gIjwyMR9VeHDeCrss1LjWMQtaVV9o+r2IZIrISSKSpqpzwx/NXXp3SuLO8wfzzhflvG4ti4xLhfQeOnixx2zgCuBZEZkS1lQudc2YvuT3TOMn81dSWWt7aeM+oZ4Uy1HVy1V1qqpeDgwNZyi38nrieHzycHb6avjL6gNOxzHmCKEW9OFXVqwHEJF+bRvH/fJ7pnPt2H4Ub6rjo7JdTscx5hChfg49WkRyAD+BjcBQEckisKdud4ffd5w3mKKPyphWVMJrt40l0etpeSZjIiDUPXQ98C7wfvDf3wP/BFaGKZerJSV4uXpYAmvKffyu2FoWGfcIdQ/9P8B0AjdabAMeUNUNIvJR2JK53PAsLxNGdOa3i9Zw8fDuDOya4nQkY0LeQ98E/FBVzwfuBu4BUNWqcAWLBg9cMpSOCR6mF5XYkyyNK4Ra0J+q6tcAqrodKAUQkXbdziMrNZH7Lsrjw7JdzFm6yek4xoRc0PtEpJeI9BCRnkCdiPQAvhvGbFHh8tG9+Eb/Tjz291WUV9iTLI2zQi3om4GHgZnAo8Apwa+vDlOuqCEiPDapgP11fh5+5XOn45h2LtSTYneo6tLDB4rIoDbOE5X6Z6Vwa+FAfvHWar41spzCIV2djmTaqZD20Ecr5uDwL0OZX4JdDmLZTWcPYFDXFO6ft4LKmjqn45h2KmwtiEQkLtiXex4wJjjsGhF5psmrZ7jWH2kJ3jhmTS5gy55qnnrLWhYZZ4St66cGWvb+UUQGEriHumH4deFap9NG9+3EVafl8Id/rmfiST0p6GUti0xkRfpROFXBJ2fMEJHxEV53RNxzwRC6pCQytegz6uxJlibCJNw364vIDOBtVX2/ybA44M/AdFVdd9j0Tftyj5ozZ05Y87WWz+cjJeXoV4d9tK2O335Sw3dzE7iwX3yEkzWfzWmWrXUKCwuXqeroFicMpdfvibyAGcDYowy/Ebi8uXmd7MvdkuZ6OPv9fr32uY90yP2v68adlZELFeTm/tKWrXVwQV/uQ0jALU0G9Qdi8lkzIsLDE4cRJ3C/tSwyERTOs9xJInI5MAq4BCgEfCJyiYjcCKxR1RXhWr/TemR05O7xufxj9dfM//Qrp+OYdiKcZ7mrgL8EX+3Sv53el7mffMXDr3zO2YOzyEhKcDqSiXH2wPcw8sQJj08uYG91LY/9PSbfXRiXsYIOs7zuaVx/Vn/mLN3M4rU7nI5jYpwVdATcfs4g+nRO4r65K9hf6/pHepkoZgUdAR3iPcy8rID1Oyr57aI1TscxMcwKOkLGDurC5JN78nTxWlZvr3A6jolRVtARdN/FeaR28DLNWhaZMLGCjqDOKYncf/FQlm3YzZ8/3Oh0HBODrKAjbPLInpwxsDM/ff0Ltu+zlkWmbVlBR5iIMPOyAg7U+5kxv122NTdhZAXtgL5dkrn93EG8vmIbC1bakyxN27GCdsj1Z/ZnSLdUHnx5JRX7a52OY2KEFbRD4j2BlkXbK/bz5AJrWWTahhW0g07OyeTq0/vy/JIylm/c7XQcEwOsoB121/hcuqV1YFpRCbXWssicICtoh6Ukenl4Yj5fbKvgv9+zJ1maExORgm4PfblPxHlDs7kwvxu/evtLynZUOh3HRLFI9+XuJyL3i8j3ReSKcK07Gs2YMIwETxz3zSuxlkWm1cJW0KrqV9U/Ap9wsC/3/cCvVfV/gHNEJCNc64822WkduOfCIfxzzU6KPt7idBwTpSL9HjpbVfcFv14NnBTh9bvaVafmMKpPJo++9jm7Kg84HcdEobD1FDuGpk219gJZh09wWF9uiouLI5PsOPl8vrBkm9zLz4Mba/nhs4u4fnjrHr8drmxtwbKFV6QLuuluJxP44vAJVHU2MBsgNzdXx40bF5lkx6m4uJhwZduWWMpvFq7hPy4cxdhBXY57/nBmO1GWLbwifci9TUQaHvg0iMD7a3OYWwoH0q9LMtPnllB9wFoWmdBFrC+3iHwTeAS4Q0SuJ/B4nH3NLqSd6hDv4bFJBWzcVcWvF4b0xF5jAGf6cs8I1zpjyekDOvOd0b2Y/e46JozoQV73NKcjmShgV4q52PSL8sjoGM/UohLqrWWRCYEVtItlJCXw4KVD+XTTHl5cUuZ0HBMFrKBdbsKIHpw9OIufvVnKV3uqnY5jXM4K2uVEhEcvy8ev8ODL9iRL0zwr6CjQu1MSPz5vMG+vKueNFdayyBybFXSU+P4ZfRnWI42fzF/J3mprWWSOzgo6Sng9cTw+eTg7fDU88cYRF9gZA1hBR5WCXul8/4x+/OmDjSwt2+V0HONCVtBR5sfnDaZnRkemFZVwoM5aFplDWUFHmeREL49els+X5T5+/4+1TscxLmMFHYUKh3TlkuHd+c3CNaz92ud0HOMiVtBR6sFLh9IhPo7pRdayyBxkBR2luqZ2YPpFeXywfhd/WbrZ6TjGJaygo9h3Rvfm1H6dmPn3VXxdUeN0HOMCVtBRLC5OeGxSAdUH6nnk1c+djmNcwAo6yg3smsIthQOZ/+lXLCotdzqOcVjEC1pEikTkmeDrPyK9/lh007j+DOyawv1zV1B1oM7pOMZBTuyh56vqdcHX0w6sP+Ykej3MmlzAlj3VPPWWPcmyPYt010+AYSJyB5AM/EZV9zqQIeac0rcT3zsth2ffX8/eQfFsSixrfoZmnk7U0nOLWnqwkTSzhNWbatn64cYW5m/9+ptbd0sL37GznnEtrNvtxKnPMEXkZOBmVb3+sOFN+3KPmjNnjhPxWuTz+UhJSXE6xiEqa5WHllRTXmWfS7dGQSflzlPd9X/aoLCwcJmqjm5pOscKGkBE3lbVc481Pjc3V0tLSyMZKWRu7eF8oM7PGwv/wRljxhwxrqX/6eb+FLS5uY9j1OLFixkTzNbSn15z62w+azPjmpnx4w8/YML4wuZDOUREQiroiB5yi0ghUK6qK0UkE7C79dtYgjeOtAShc0rrnroRbpkd4shO6+B0jKNakxj9D0mN9Hvo5cAPRCQfGAJMj/D6jYlpES1oVd0D/CKS6zSmPbELS4yJIVbQxsQQR89yt0REKgB3nuaGLsAOp0Mcg2VrHTdny1XV1JYmcuLCkuNRGsqpeieIyFLLdvwsW+uIyNJQprNDbmNiiBW0MTHE7QU92+kAzbBsrWPZWiekbK4+KWaMOT5u30O7mkhL9x0ZE1muLWgRuVVErhKRB0Skk9N5GohInIhMEZF5wJF3QDhIRDJE5M5gvhdFZLDTmRqISKqI3Cci3xOR2SIy0OlMhxORNBF5w+kchzuepiCu/NhKRIYA2ar6nyKSC9wBPOBwLABU1Q/8MfgH6bY9dB6wQFVLRGQLgdtQ73I4U4Ms4F1VfU9ENgI3Anc7nKlR8GjrW4Ab+zjNV9XnQpnQlQUNnAksA1DVUhEZ6XCeqKCqS5p82xn4xKksh1PVdcC64Lf9gHcdjHM0lwKvA2c7HeQoQm4K4tZD7i7AvibfxzsVJBqJSBKBjeJfnM7SlIh0E5FfEjiSeN3pPA1EZCiwR1VdeTuvqt6tqk8BrwE/b25atxZ0OZABjYdCB5yNEz2Cv687gIdU1VXNulV1m6r+CPg98DOn8zRxCTBGRKYCBSIyNbhRdBVVXU7g6OaY3HrI/R5wDVAEDAVCuuzNAHAt8Jyquup5s8G3TeWquhmoAEY4HKmRqj7R8LWIDFHVx53M09TxNgVxZUGr6moR+UpErgH6Ar9yNtFBwS33xcAooIOIJKjqQodjASAik4ErgbHBT9RqVPVGZ1M12gJMEZGdBH531twiNMfVFMQuLDEmhrj1PbQxphWsoI2JIVbQxsQQK2hjYogVtDExxAranBARGSciT7Q8pYkEK2hzQlS1GLBr7V3CCtqYGOLKK8VMeIjIbcAeoA+Bu50mAR4Cd7YNAjaq6u9FJAW4DVgLZAMfqOoHIpJF4JbM1cAA4L9VdWdw2eOBk4EJwAWqug8TcVbQ7YSIfBPoqKq/Dt7A8QLwLHBWw722IvJXEfkr8GPgJVVdEZz2ZRGZCMwCpqtquYj0BLoBOwkc6X2qqm+KiB84HXgz0j+jsYJuTxquPb+MwF55Q3C4v8k0awnsvUcAMwBUVUWkGkgH+qhqeXD4FgLXZwP4m9x6WE6gmYFxgL2Hbj9WARWqOk9V/0awYDm068oAAk0ISoDB0Hg7ZkdgL7BDRAY1TCwiaRHIbY6D3ZzRTgQL82EgAdhMoGgBbgIWEbirbZWqvhAs1NsIFHc28C9VXSIiOcBjwJfALuD/gBzgeeCHBDqkzATqgWmqWhGZn840sIJux0RkHDBWVR91OotpG3bI3U6JSDKBrqVDRWSA03lM27A9tDExxPbQxsQQK2hjYogVtDExxAramBhiBW1MDLGCNiaG/D/3lO2Tdx60MQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.0001, 5, lang, device, first_train=False)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.002, 10, lang, device, first_train=False)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.002, 10, lang, device, first_train=False)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def truncate_pad(line, num_steps, padding_token):\n",
    "    \"\"\"截断或填充文本序列\"\"\"\n",
    "    if len(line) > num_steps:\n",
    "        return line[:num_steps]  # 截断\n",
    "    return line + [padding_token] * (num_steps - len(line))  # 填充\n",
    "\n",
    "def predict_seq2seq(net, src_sentence, lang, num_steps,\n",
    "                    device, save_attention_weights=False):\n",
    "    def voc(line):\n",
    "        return [lang.word2idx(i) for i in line]\n",
    "\n",
    "    #\"\"\"Predict for sequence to sequence.\"\"\"\n",
    "    # Set `net` to eval mode for inference\n",
    "    net.eval()\n",
    "    src_tokens = voc((src_sentence.lower().split())) + [3]\n",
    "    enc_valid_len = torch.tensor([len(src_tokens)], device=device)\n",
    "    # 截断、填充文本序列\n",
    "    src_tokens = truncate_pad(src_tokens, num_steps, 1)\n",
    "    # Add the batch axis\n",
    "    enc_X = torch.unsqueeze(\n",
    "        torch.tensor(src_tokens, dtype=torch.long, device=device), dim=0)\n",
    "    enc_outputs = net.encoder(enc_X, enc_valid_len)\n",
    "    dec_state = net.decoder.init_state(enc_outputs, enc_valid_len)\n",
    "    # Add the batch axis\n",
    "    dec_X = torch.unsqueeze(\n",
    "        torch.tensor([2], dtype=torch.long, device=device),\n",
    "        dim=0)\n",
    "    output_seq, attention_weight_seq = [], []\n",
    "    for _ in range(num_steps):\n",
    "        Y, dec_state = net.decoder(dec_X, dec_state)\n",
    "\n",
    "        dec_X = Y.argmax(dim=2)  # 最大的下标，即为最终词下标\n",
    "        pred = dec_X.squeeze(dim=0).type(torch.int32).item()  # 去掉第一维取结果\n",
    "        # Save attention weights (to be covered later)\n",
    "        if save_attention_weights:\n",
    "            attention_weight_seq.append(net.decoder.attention_weights)\n",
    "        # Once the end-of-sequence token is predicted, the generation of the\n",
    "        # output sequence is complete\n",
    "        if pred == 3:\n",
    "            break\n",
    "        output_seq.append(pred)\n",
    "    return ' '.join([lang.index2word[i] for i in output_seq]), attention_weight_seq\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " what ’ s wrong with that ? \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i ’ m not sure . i ’ m doing it . i ’ m going to have a lot of business .\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " what ’ s wrong with that ? cigarette is the thing i go crazy for .\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i ’ m not sure . i ’ m doing it . i ’ m going to have a lot of business .\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " so dick , how about getting some coffee for tonight ?\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yes , i am . i am very happy to see you .\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " q\n"
     ]
    }
   ],
   "source": [
    "while 1:\n",
    "    sentence = input()\n",
    "    if sentence == 'q':\n",
    "        break\n",
    "    out, weight = predict_seq2seq(net, sentence, lang, num_steps, device)\n",
    "    print(out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_sampling(net, data_iter, valid_iter, lr, num_epochs, lang, device, first_train=True, min_ppl=1e9, tg=0.5, tr=0.95, i=70, lloss=-1,lppl=-1):\n",
    "    # \"\"\"训练序列到序列模型\"\"\"\n",
    "    def xavier_init_weights(m):\n",
    "        if type(m) == nn.Linear:\n",
    "            nn.init.xavier_uniform_(m.weight)\n",
    "        if type(m) == nn.GRU:\n",
    "            for param in m._flat_weights_names:\n",
    "                if \"weight\" in param:\n",
    "                    nn.init.xavier_uniform_(m._parameters[param])\n",
    "    lang_len = len(lang)\n",
    "\n",
    "    net.to(device)\n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=lr, betas=(0.9, 0.98), eps=1e-9)\n",
    "    net.train()\n",
    "    animator = Animator(xlabel='epoch', ylabel='loss',\n",
    "                        xlim=[0, num_epochs])\n",
    "    animator1 = Animator(xlabel='epoch', ylabel='ppl',\n",
    "                         xlim=[0, num_epochs])\n",
    "    LOSS = PPL = 0\n",
    "\n",
    "    timer = Timer()\n",
    "\n",
    "    if first_train:\n",
    "        net.apply(xavier_init_weights)\n",
    "    else:\n",
    "        net.eval()\n",
    "        PPL_AVG = 0\n",
    "        LOSS_AVG = 0\n",
    "        with torch.no_grad():\n",
    "            #             i = 0\n",
    "            for batch in valid_iter:\n",
    "                # 转device\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                # inputs带<eos>不带<bos>，outputs带<bos>不带<eos>\n",
    "                # 初始bos\n",
    "                bos = torch.tensor([2  # lang.word2index['<bos>']\n",
    "                                    ] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)\n",
    "                # Teacher forcing, 输入X和正确答案，希望得到正确答案\n",
    "                # Y_hat\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                # Y_hat和Y中都有eos，没有bos\n",
    "                l = loss_fn(Y_hat, Y)\n",
    "                ppl = torch.exp(l)\n",
    "                LOSS_AVG += l.item()\n",
    "                PPL_AVG += ppl.item()\n",
    "            PPL_AVG /= len(valid_iter)\n",
    "            min_ppl = PPL_AVG\n",
    "            LOSS_AVG /= len(valid_iter)\n",
    "            if lloss == -1:\n",
    "                lloss = LOSS_AVG\n",
    "            if lppl == -1:\n",
    "                lppl = PPL_AVG\n",
    "            animator1.add(0, (lppl, PPL_AVG,))\n",
    "            animator.add(0, (lloss, LOSS_AVG,))\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        #         lr = 512**(-0.5) * i**(-0.5)\n",
    "\n",
    "        for batch in data_iter:\n",
    "            net.eval()\n",
    "            with torch.no_grad():\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                bos = torch.tensor([2] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 强制教学\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                p = Y_hat.softmax(2)\n",
    "                # p = F.cross_entropy(Y_hat.permute(0,2,1), Y, reduction='none')\n",
    "                M, I = torch.max(p, dim=2)\n",
    "                Y0 = I*((M>tg)*(M<tr)) + Y*(M<=tg) + (M>=tr)*torch.randint(0,lang_len,Y.shape,device=device)\n",
    "\n",
    "            net.train()\n",
    "            optimizer.zero_grad()\n",
    "            bos = torch.tensor([2] * Y0.shape[0],\n",
    "                               device=device).reshape(-1, 1)\n",
    "            dec_input = torch.cat([bos, Y0[:, :-1]], 1)  # 强制教学\n",
    "            Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "            l = loss_fn(Y_hat, Y)\n",
    "            ppl = torch.exp(l)\n",
    "            l.backward()\n",
    "            #             grad_clipping(net, 20)\n",
    "            optimizer.step()\n",
    "            with torch.no_grad():\n",
    "                LOSS += l.item()\n",
    "                PPL += ppl.item()\n",
    "        # i += 1\n",
    "        #             if i == 30:\n",
    "        #                 break\n",
    "        LOSS /= len(data_iter)\n",
    "        PPL /= len(data_iter)\n",
    "\n",
    "        net.eval()\n",
    "        PPL_AVG = 0\n",
    "        LOSS_AVG = 0\n",
    "        with torch.no_grad():\n",
    "            #             i = 0\n",
    "            for batch in valid_iter:\n",
    "                # 转device\n",
    "                X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]\n",
    "                # inputs带<eos>不带<bos>，outputs带<bos>不带<eos>\n",
    "                # 初始bos\n",
    "                bos = torch.tensor([2  # lang.word2index['<bos>']\n",
    "                                    ] * Y.shape[0],\n",
    "                                   device=device).reshape(-1, 1)\n",
    "                dec_input = torch.cat([bos, Y[:, :-1]], 1)\n",
    "                # Teacher forcing, 输入X和正确答案，希望得到正确答案\n",
    "                # Y_hat\n",
    "                Y_hat, _ = net(X, dec_input, X_valid_len)\n",
    "                # Y_hat和Y中都有eos，没有bos\n",
    "                l = loss_fn(Y_hat, Y)\n",
    "                ppl = torch.exp(l)\n",
    "                LOSS_AVG += l.item()\n",
    "                PPL_AVG += ppl.item()\n",
    "            #                 i += 1\n",
    "            #                 if i == 10:\n",
    "            #                     break\n",
    "\n",
    "            PPL_AVG /= len(valid_iter)\n",
    "            LOSS_AVG /= len(valid_iter)\n",
    "            if PPL_AVG < min_ppl:\n",
    "                if min_ppl != 1e9:\n",
    "                    torch.save(encoder, MODEL_ROOT + \"trans_encoder.mdl\")\n",
    "                    torch.save(decoder, MODEL_ROOT + \"trans_decoder.mdl\")\n",
    "                min_ppl = PPL_AVG\n",
    "\n",
    "        if (epoch + 1) % 1 == 0:\n",
    "            animator1.add(epoch + 1, (PPL, PPL_AVG,))\n",
    "            animator.add(epoch + 1, (LOSS, LOSS_AVG,))\n",
    "\n",
    "    print(f'loss {LOSS:.3f}, PPL {PPL:.3f}, {timer.stop() / 60:.1f} '\n",
    "          f'min on {str(device)}')\n",
    "    print(f'loss {LOSS_AVG:.3f}, PPL {PPL_AVG:.3f} ')\n",
    "    print(lr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 2.523, PPL 12.654, 10.9 min on cuda:0\n",
      "loss 4.039, PPL 57.559 \n",
      "0.001\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAC4CAYAAAAhS0pZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3Xl8VNXZwPHfM5NJMslk3wlbgBCWhB1lcQl1ra37brWCC671rdrFttba2r61+lar1pa6FFzaWlTU1roAFiqVRREQwpKEnQSSEDJJJsvMJJnz/jGTEEIIWWbnfD+ffDKZe+fOM0mee889qyil0DQtfBkCHYCmab6lk1zTwpxOck0LczrJNS3M6STXtDCnk1zTwpxOck0LczrJNS3M6STXtDCnk1zTwlxEoAPorcTERDVq1KhAh9GtxsZGYmNjAx1Gt3Rs/RPMsX355ZfVSqm03u4fMkmekZHB+vXrAx1Gt1auXElhYWGgw+iWjq1/gjk2EdnXl/11cV3TwpxOck0LczrJNS3Mhcw9uaaFIudhJ5sKNxGRGIFlsoWUi1JIuSjFrzHoJNc0H6r5oIambU3ETYuj8pVKJEJIuSgFl9PFhpkbiM2PxTLZQtzkOGInxmJKNHk9Bp3kmuZDNUtrMGWYmLJuCgCuZhcArdZWIjMisS61UvlqZcf+uc/nkn13Ni21LdT9tw7LJAtR2VGISL9j0EmuaT6U86scsm7NQgzuJDXGGgGIzIhkwgcTAHBUOGjY1EDDpgYSZicAUL+mnqKLiwAwpZqwTLZgmWRh0F2D+hyDT5NcROKBxUqpC7s8nwN8CygHmpVSb/gyDp8rBgoDHYQWjMzDzZiHm3vcJyoziqgLo0i58Oi9euLZiUz+72RsG23uE8DGBsqeKSPjpow+x+CzJBd3+eJKoKqbzQ8D9yul6kXkRRH5SClV66tYfOnIR0fgTjjIQQbd0fezrBa+qv9RTcuRFjLnZva5uG2MMZIwO6Hjyg7ganEhxr4X233ZhHYx8CHg6mZbhlKq3vO4BJjkwzh8xr7fjmWSBU6H0ntLqV0VkucpzUfKni6j7JmyAd1Pd2YwGTqK/X3hkyu5iIwDapVSFSf4gJGdHtcB3fbDFZH5wHyAtLQ0Vq5c6eVIB+gJ4DNoWNCA5YcWNl2yCRYAfS9R+UxDQ0Pw/d48wjq2ZuC/wFUE/DP6qrj+TcAgIrOAAhF5CHhWKdXk2e7stG8SsKO7gyilXgBeAMjLy1PB1JdYKcXab68l7tw4yILpS6ez4fQNmJ8wM2XtFAym4OhnFMx9sMM5tur3qylqLWLibRNJKkzyXmD94JMkV0o90f5YRMYopR4XEbOIGJRSLqBCRBKUUnVALvBHX8ThS80lzTgOOBj646FUU03smFjG/mUsLVUtQZPgWuBYl1oxmA0knJFw8p19zJ9NaE8Aq4DFwGPA/SJSDizvdH8eMqzLrQAkn5cMB9zPpX4ztWO7s9pJZGpkdy/VTgH2/XYS5yRiiAr8Cd/nSa6Umuv5/p1Oz+0DHvX1e/tSzbIaonOiMY80dyR5u9pVtWz++mbGvzX+mGYR7dRR8G4BLkd3dc7+F/jTTIga9dQoxiwa0+22uClxmEeZ2XbdNppKmrrdRwt/wXAVB53k/WYeYSbxrMRutxljjeS/m4/BZKDo0iJa61v9HJ0WSDtu2UHJXSWBDqODTvJ+OPzOYQ4tOtTjPubhZsa/NZ7mnc1s/9Z2lEuvHnsqcLW6qH6nGldLcBTVQSd5v5Q9VUb578tPul/i2YmMemYU0SOjdZKfImxf2GitbSX5/ORAh9JBD1Dpo1ZbK/Vr6xnyvSG92j/77uyOx6pN9atbohY6rEutIJB0bmDbxjvTV/I+qv1PLapVkXRe3/6IDVsa+Hz859g22XwUmRYMaj6uIW56HKZk748L7y+d5H1kXWbFEG0gflZ8n15nSjPR1tBG0WVFOA87T/4CLeQo5T75Z92WFehQjqGTvI8c+x0knJ2AMdrYp9dFZUaR/24+LZUtbLtmW1BVzGjeISLk/DyHQbcH12hEneR9lP9OPgXvFfTrtfHT4hn94mhqV9ay64FdXo5MC7TG7Y202dsCHcZxdJL3w0A6OWTemMngBwfTVNwUND2itIFTSrH565vZcVO3Y60CSteu90HJvSUohyLvxbwBHWfE4yMAMEToc2y4aC5txrHPQeJD3XeQCiT9X9ZLSimq366mrWngxTFDhAFDhAFHhYOiK4qwl9m9EKEWSDUf1wAEVft4O53kvdRY1IizwunV9s/Wmlasy6xsvXwrbc3Bdy+n9Z51qRXzKDPmET3P5xYIOsl7ybrMPbTUm0keOy6Wsa+PxbbeRsn8EpTSveJCkcvpwrrCStL5wdMBpjN9T95L1mVWzHlmoodEe/W4qZemMvwXw9n7yF4sky0MeaB3Pem04CERwqRPJmFM6Fuzqr/oJO+l+Nnxfe7l1lvDfjKMhq8aOPjCQQbdPajPbfBaYIlBiD+9b52j/EkneS8Nf3i4z44tBmHMojGoFqUTPATte3wfiWclkjAr8FM9dUffk/dC855mn3dyiLBEYEoy0drQSv36kJsN65TlPOxkz4/2ULsieKfj1kneC9uu28aWr2/xy3vtuHkHRRcX4XLqjjKhoKNCNkgr3UAn+Um1WFuwrbeRcLZ/imJZt2XhrHBy+O3Dfnk/bWBqltYQkRJB3JS4QIdyQjrJT6L237Xg8t/44OQLkjHnmil/7uSTUmiBpZTCutRK0rlJQT1PgE7yk7Aut2KMM/qt9lQMQvY92dSvqdf35kHOedCJcimSLwi+Xm6d6SQ/iZplNSQWJvp1wYTMuZkYLUZqPqjx23tqfReVHcWsQ7PIuDGI1sXqhm5C64FSirGvj+3XInMDEZEQwWk7TiMqO8qv76v1nYggpuAtqoO+kvdIREiYkUD8af7v6NCe4HpyieDU1tzGujHrqFrc3crcwUUneQ8OLTyE9d/WgL3/gd8d4POxn+tED0J1n9bRXNyMMT74Oy/5JMlFJE5EfiIiN4jICyIyqsv2QhH5u4i85Pma6Is4BkK1KXZ9bxeVr1UGLIaY3Bjsu+xUL6kOWAxa92qW1iBRcsIFNoKJr+7J04BPlVKrRGQ/cAfw/S77/FAptddH7z9gto02WmtafdZfvTeSv55M9Mhoyp4rI/3a9IDFoR2v5uMaEs9MxBhzil7JlVK7lVKrPD/mAJ922cUB3CAiPxaRG3wRw0C1r1qadE7gkryjOe2zemwb9FTOwcJR7qBpaxNJFwRvL7fOxFdjmEUkE3gIaAIeUUp1uyCYiDwNLOl0Uui8bT4wHyAtLW3q4sWLfRJrtx4A6oCXT75rQ0MDFovFN3E0AFcDhcAP+/FyX8Y2QCEbWwXwOnAVMNx/MbWbM2fOl0qpab1+gVLKp1/AMODpHrZfAHz/ZMcZPXq08hdXq0utHrpalT5Y2qv9V6xY4dN4Kt+sVI0ljf16ra9j64+2ljZVen+pWvH6CqWUUnVr65SjyhHYoLoIxt9bO2C96kMO+uSeXESmAFVKqTLABkwUkUjPScUpIvcAf/AEPBLY6os4+kuMwow9M3A1B0etdvpV4XU/XrGogrKny+BR90Vm23XbsB+wk3ROEunXp5N6WSqmxOBZgaQz5VI0bmkkdkIsIsHdPt7OV01o5cC1IjIX+DnwY+A+4H882/cDV4jIPCAW+NBHcfSbGARjbPBUqtg22Nhxy46Qb05ra2xj7yN7iZ8ZD2e5n8t/L5+hPxhKc2kzxfOKWZ2xmgO/PRDYQE/AtsHG+knrObw4dAYQ+eRKrpSqBH7r+XGR5/vaTtv/6Yv39ZYtl2whcU4iQ+4PnqmYHOUOKhZWkPz1ZNKvDt0r+4GnDuA85GT8W+PZ6NyIiGCZYMEywULOr3KwfWGj6o0qYifEAtBU3MTeR/eSfl06yRcmD2jOe2+wLnVXyCbOCf6ms3a6W2sXzionR/55hPgZwTWdT8pFKUTnRFP+XHnIJrmz0smBJw6QekWqexaVlcduFxHiT4s/podhU2kT1uVWqt6owphgJO3yNNKvSyfxnMSAzFtf83ENlskWItMj/f7e/aV7vHVh/cTTdBbA9vHuiNHdnFa3qi5kV0YVk5B5SyYjfj2i169J/WYqMw/OZMJHE0i7PI3DSw6z5ZIttDW4Z+pxHHL4be33Vlsr9avrg37UWVc6ybuwLrMSkRSckwBk3pKJIcYQsmPNTckmcp/JJWZ0TJ9eZzAZSL4gmTELxzC7ajaT/jOpo2Ju6xVbWTNkDTsf3ElrfbettF5Tu8KzbHUQzwLTHZ3knSilsC63kvi1xKCcBMCUZGLIA0Mw5wbfBP4ns/snu6ldNfB50AxRBhJmuGfpUUox+LuDiT8tnrJnyth61VafVkwmfi2R/H/kB+2EjSei78k7cdldJJ2fRPJ5wVscy3ksJ9Ah9Fntqlr2/+9+jLFGEs/0XoWViJB+bTrp16ZzaNEhiucVU3JXCXkv5vmkeSvCEkHqxaleP66v6STvxGg2MualMYEO46RcrS5qPqgh+aLkoF80USn3QJ/I7EgGf3ewz94na24W9t12mrY3oVoUEundJLcfsFOxqIKsW7OIGhRa4/yD+z/Ez+z77CGxVJH1YytFlxZx5L0jgQ7lpA4vPoztcxs5j+X4fDDH8J8PZ9zfx2GINHi9Mu7Iv46w95G9tNlCb806neQerhYXXxR8wa4HdwU6lJNKvjCZ6OHu0WnBzOVwsftHu4ktiCXz25k+fz8RQQyCvczOl6d9Se2n3psL3brUStSwKMyjQ68+RCe5h+1zG222NuJnBVf7eHfEKAy6ZxB1/6mjYXNDoMM5MYHs72Qz6ulRfq3INMYYaWtoo+iyIpqKmwZ8PFeLC+snVpLPTw6Zrqyd6ST3sC63gkDS10KjeSTrliwM5uBuTjNEGhhy/xC/D9c1JZuY8MEEJELYfNFmnIedAzqe7XMbbfVtIdd01q5XSS4i80RknIgUeGZ0mevjuPyuZlkNcdPiMCUH58CIrkzJJjJuzKD+83pUW/DVI+z/v/1UvFYRsPc3jzCT/498nAedFF1SNKD135t3NmO0GAM6t8BA9PZKXqeU2oZ7RPOtgHfX7w2w1vpW6tfW+20BBW8Z+duRTNs4Leja9O377Ox5eE/HxBuBkjAjgbGvj8XlcNFa1/+OMpk3ZzL7yGxMSaFxAeiqt01oCSIyA9ivlGoQkeCv1u0DQ5SB/HfyQ66TSUSc+8/X1tyGIdIQNMm+5+E9iAg5vwx8m37alWmkXJqCIcKAUqrf99SGyNC9s+1t5F8BFwK/EZFJQJbvQvI/Q5SB1ItTiR0TG+hQ+qxxWyNrBq+h+h/BMdmjbYONytcrGfzdwUQPCY4CnyHCQFtzG0WXF1H+x77VYRx++zBfTv8Se5ndR9H5Xq+SXCm1QSn1qFKqDmgEFvg2LP8qf76cxm2NgQ6jX8yjzRgtxqCogFNKsev7uzClmhj60NBAh3MMMQmqVVF6bylHPuh9QbTmoxqaSpuIzAydUWdd9bbi7VFPxdttwF3AT30blv/Yy+yU3ltKzYehuSSRIcJA9j3Z1K6opWFL4JvTsu/NZtSzo4hICK7OlIYIA+PeGIdlooWt12zFtvHkI/mUUtR8XEPSOUlB37OwJ72NfDOwA7hQKfUAsNN3IflXx6ysIVbp1lnWrVkYog2U/z6wV3MRIe3yNDKuD861wSIsERS8X4Ap2cSWb2zBfqDnInhTcROOA46QG1raVW+TPAd4DlgsIhYg+MZh9pN1uRVTuonYgtC7H29nSjGR/q10Kl+rpKW2JSAxVLxewZ5H9wT99FRRg6Io+FcBxlgjzsqe28+tH3suACHaPt6ut2Wqp4GhSqm9ntVQvvBhTH7TPrQ06dwkvy9q6G1DHxpK1i1ZASkmtzW2sfsHu4keHs3wnw33+/v3laXAwvTt0zuK4CeqdTfnmsm6Iwvz8NBqdemqt/8RCjhbRO7DXdP+qu9C8h/7XjttdW0hXVRvFzMqBkadfD9f6DxvW6h0+2xvUtv7yF6cFU5GvzD6uNhTLkoh5aKUAEXoPb0trn8P99TKL+BecqDrkkchyZxjZnbNbNKvC80507pqqW2h+M7iPtUeD9Rx87aFkPakPvTSIfb/Zv+xG6vBcdARgKi8r7dJfkAptUQptUMp9S7uKZfDgtFsxGgOnqmXB8JoMVLzrxoOPOW/6Yz3ProXl93FiMd7P29bMBn+i+Gk35DOnh/tofKNTotbLoG1w9fS1hR6Q0u76m2Sd10vJnRrqTxcDhcbz9pI9fvB0YnEGwwRBgbdPYjaT2pp3Oqfdv+s27MY9ewoYnL7Nm9bsBARxvx5DAlnJrDj5h3U/tczPPULiJ8VHxILGp5Mb5P8kIgsEpFfiMhC4JAvg/KHutV11K2qc9c2hJGs27OQKPFbc1rclDiy78r2y3v5iiHKQP67+cSMjsGx3+Gudd9JyDedtettj7d/4V795D3gfp9G5CfWZVYwQuLZoTNJfm9EpkaScUMGFa9W+LQ5rXZVLdvnbqelJjBNdt5mSjYxdeNUMm7IoGaZu2NU8vnhkeQnrF0Xkd8AUzj+WidAJnDCVVBEJA73skh7cK/H+YRSamen7SnAvbg71SQopf7Qz/j7zbrcSvyMeCLig6tnljdk35eNIcqAcvimmNI+b5uj3IEhOnR7gnXV3qRW+4m7yG6ZHJwrrvZVT//hHyilul0sV0RmnuS4acCnSqlVIrIfuINja+QfBBYqpUpF5FcikquUKu1T5APQUtOCbb0tJNp0+yNuUhxxf/T0V9ru/eO3z9uWtzAvLO5Zu8q6I4uKgoqQ7zvR7oSnYaXUf3rYtqangyqldquj643nAJ922WVCp6TeBMw+aaTNeK1o2GptJeWSFJK/Hh7Fse4opaj7rA6KvHvcjnnbJsSSeZPv520LhIQZCe4ybJgQX81OKiKZwENAE/CIUqq107alSqnzPY/PByYqpZ7s5hjzgfkAoxk99U/8CZJwr3g+DPgm7g4gLtw3EQE68fa4YH2gtAE3AFW4f2c5wAhgFjB5AMd9C3geeAKYPrAQg/L35hHMsc2ZM+dLpdS03u7vsxtSpVQF8F0RGQY8ybEVdp07DScBlXRDKfUC7g445A3OUyPvH0njtkaatjXRuLKR8XePJ7kwmer3q9lx0w5ixsYQMy6G2HGxxIyNIeHMBCIsx39EZ6WTyAzvDR1cuXIlhYWFXjuetzSvbWbdb9eR2ZxJ4+ZGGv/VyNAJQxleOBxntZONszcSWxCLpcBC7IRYYgtiMY8w91hMdeQ6qBpRxZAHBr7ia7D+3iC4Y+srnyS5iEwBqpRSZbh7yk0UkUgApZQT2CgiY5RSO3BfV1446UFjYciDR/+xlFIdVYJRWVGkX59O47ZGjvzzCBUvu+cWm751OhHjIjj8zmGq36kmZmwMkRmRFN9azJhXxvhlmuBAMueY4SoYU+heMEK1KVwO9wASV6OL2PxYGr9qpHpJdcfvMu/lPLJuycK+3071u9XEFriTPzLVfVKMyo7ySoJr/uOrK3k5cKNnmqipwI9x17YL7qv607iv8rtwTym1u69vICIdxfO4qXHETT06MM5Z7aRpe1PHdE7Og05qV9RS+drRAkOwLU3sD2KUjoqy6GHR5L+dD7gHmDRua6RxcyOJX3M3KdavqWfn/xwdURyREoEp1UTBewXE5IVmx5dTlU+SXClVCfzW8+Miz/e1nbbXAI/44r3B3VYceebR4nj2Pdlk35NNa30rTTuacDlcfV5ZM5wZY43ET48nfvrRE1/aNWnMPHumu5i/pZGGLQ04DzkxWsKvNj3chV8jcQ8i4iOOWeBeOzERISoziqjMqLDpFHKqCp+eDJqmdUsnuaaFOZ3kmhbmdJJrWpjTSa5pYU4nuaaFOZ3kmhbmdJJrWpjTSa5pYU4nuaaFOZ3kmhbmdJJrWpjTSa5pYU4nuaaFOZ3kmhbmdJJrWpjTSa5pYU4nuaaFOZ3kmhbmdJJrWpjTSa5pYU4nuaaFOZ3kmhbmTql514OZs9XFm18eYFVJNVdOHcy5Y9Pdq8SEGaUUq0qreXXNPnYfbOaD6q8YnRFHXmYcozPiSI+LCsvPHUg6yQOspc3FOxvKefbfpZRZm4mLjuCjrRVMGpLIDy7IY9ao1ECH6BVNzlaWbChn0eq97KxqINUSRWok/HtHFYvXl3Xsl2A2kZcRR26GpSPxR2fEkRzrvQUqTzW+WvAwEbgV92qlFwCPKaVKOm0vBO7CvRgiwHNKqa98EUuwam1z8d6mgzz771L2HWli4uAEfnlZPrNHpbJkQxnPLC/lhpfWMWtkCt+7II8pQ5MCHXK/lNc28+rqvbzxxQHqmlsoyE7gqWsm8o0JWaz57yoKCwupbnBQUmmjtLKB4kobJRU2/vnVQf6yrmO1a1ItUeRlWshNj+uU/Bbiok0B/HQnZ29p49U1e6ltaunV/r4oxPjqSj4WWKqU2iIi5bjXGP9el31+qJTa66P3D1ptLsX7mw/yzCel7D7cyLiseF769jTO6VQ8v3b6UC6dlM1f1+3n+RU7ueIPqzl3bAYPnj+asVnBv8yTUor1+6ws/GwPHxW5V5i9MD+TW2bnMHVY0nHF8VRLFKmWKGaNTD3mGJX1DoorbZRW2iiusFFSaePvXxyguaWtY79BCdGMzoxjXFY8t505Iqiu+DZ7C7e9sp51e2owGU+evUqd/Ji92OU4vlrwcE2nH1OATV12cQA3eP7Ye5VSf/VFHMHE5VJ8WFTB75aXUFrVQF5GHAtunML54zIxdLMeeLTJyC1n5HDt9CEsWr2XBf/ZxUXPruLiCYO4/7zR5KTGBuBT9MzR2sb7Xx1i4eo9FJXXk2A2cftZI/j2zOFkJ5r7dCwRITMhmsyEaM4endbxvMulKK9tprjCdvQEUNnAqtLdfFhUwaJ50xmWEvjfzZEGBzcv/Jwdh2w8c90kLp2U7bVjy6/7uL/qzemjn0QkBvg18AOllOME+zwNLFFKrepm23zcpQDS0tKmLl682GexDkRDQwMWi6XbbUopNlS18e7OFg7YXAyKFS4bFcm0TCOGPpTNGlsUH+5pYem+FlpdcEZ2BJeONJFi7rmBpKfYvKXW4WLF/lZWHGil3qkYFCucN8zErEERREWc+DN6M7YSaxvPbLBjELh/SjQjEge2+upAYjvS7OLJ9XaONCvunRzFxDTvXkvnzJnzpVJqWm/391mSi/sy/WPgj56lik+03wXABKXUkz0dLy8vTxUXF3s5Su9YuXIlhYWFxzynlGJFcRVPLSuhqLyenNRY/uecXC6eOAhjN1fu3jpsc/D8ip38dd1+AG6cMYy754wk1RLV69i8ZUtZHQs/28P7mw/hbHMxJy+NebNzODM3tVc15N6ObdfhBuYu/JzDNgfPXT+F88Zl9PtY/Y1t1+EGbnppHTZHK3+eO53pw72/IqyI9CnJfVm7fiuwqD3BRSQSQCnlFJF7gD8o9xlmJLDVh3H4lVKKT0ureWpZCV8dqGVIspknr5rA5ZOziTAOvFtCWlwUj14yntvPGsGzy0tZtHoPb3yxn1tm53D7WSNIMPu2Iqq1zcXSbZUs/GwPX+y1EhNp5PrThnDzrOGMSPNtieFkRqZZWHLXbG595QvueG09P79kPDfNHO639y8qr+Pbf/4cg8Ab82cwflCC3967J76qXb8CuB44w3NGdwClgABPAvuBK0QkHogFPvRFHP62eqc7udfvs5KdaObxKwq4cupgTF5I7q6yE8385qoJzD97BE8vK+H3K3by6pq93Fk4krmzhhMTOfA/rbPVxZFGB9U2J4cb7Gw/ZOOv6/ZTXtvM4CQzD39jLFdPG+LzE0tfpMVF8cb8GXznrxv56XtbKatt5ocXjOm23sOb1u4+wm2vrCfBbOL1204PqjoTX1W8LQGW9LD9n3095gGbixtfWseodAu5Ge6mlNx0C0lBUJtaXNPGghfWsHZ3DZnx0Tx2WT7XTBtMVMTA7gt7Y2Sahd/fMIW7Cut4amkJT3xUzJ//u5d754zk+tOHHrd/m0t1SlwHh20Oqj3fj3nc4Oi22WfGiGQeuXgc547NGNBthy/FREbwp5um8rN/bOVP/9nNwVo7/3f1BJ/9PT7ZXsndf9nAkOQYXrv1NLIS+lbJ6Gsh0xnGHCHY7C0sXn+AJufRJpRUS6Q78dPdHSjaH6daIr3Sc0opRX1zKxX1dirr7e7vdfaOn8uszeyosJNqUfzs4nFcf9pQok2+T+6uxg9K4OW50/lyn5UnP97Bo//cxour9jAspoWXd63zJLCTmkYHrm6qYcwmI2lxUaTFRTEyzcLpI5JJs0STGhdJmsX9fFaCmcyEaL9/tv6IMBr45WX5DE6K4Tcf7aCy3s6LN00jIca7pY53N5bz4JtfMX5QPIvmnRZUTXjtQibJU83Ce/eegVKKg3V2Sitt7KxqoLSygdIqG+9uKsdmP9p5IjHGRG66hVGeK35uhuW4bpPOVheV9XaqbHYq6hxHE7nO/b09qe0truPiSYoxkREfTUZ8NJMSHPzsW3MwR/o/ubuaOiyJv90+g892HuF3y0sorrQz2NjK4KQYJg9NIs0SSVqcu126PalTLVHERoXMv0KviQh3FY5kUGI033vzK65csJpF86YzOCnGK8d/ZfVefvaPrcwckcKLN0/DEqS/w+CMqgciQnaimexEM4V56R3PK6Wosjk6kr60qoGdlQ18WHSIv3UqdsZFR5CVEM2RBidHGp3HHT8ywkBmfDQZ8VHkZydw7tgMMhPcyZyZEE1GXDTp8VHHXK1XrlwZFAneTkQ4IzeVM3JTPbXEswMdUkBdOimb9Lho5r+2nsv/sJqFc6eTn93/SjGlFM9+spOnl5dw3rgMnrt+ckBKb70Vckl+IiLScWU9I/fYnlPVDU5Kq45e+Svq7UwdltyRzBkJ0WTGu78SY0x6gEQYmjkyhbfvmsW8hV9wzZ/W8Py3pjCn00Wit1wuxWP/2sbCz/Zy1dTBPH5FgVdaTXwpbJL8RESko1jaudukduoZnRHHkrvdiX7bK+v51WX5XHfa8ZWTJ9La5uIHb29myYZybpmdw8MtDfbsAAAF6ElEQVTfGOvzWntvCO5TkKZ5WUZ8NIvvnMnsUak8tGQLv11aTG86hNlb2rjz9Q0s2VDOg+eN5qffDI0EB53k2inIEhXByzdP49ppQ3ju3zt5cPFXOFuPr1xtZ7O3MG/hFyzfXskvLh3Pd87JDalburAvrmtad0xGA49fWUB2kpmnlpVQabPzxxunEt9l6GpNo5O5Cz9n28F6rw808Rd9JddOWSLCfefk8n9XT2Td7hquWbCGQ3XNHdsP1jZz9YLVFFfYeOHbU0MywUEnuaZx1dTBLJw3nTJrM5c/v5rth+qpaHRx9YI1VNU7ePWW0/jamP4Pdgk0neSaBpyZm8abd84E4OoFa/jVumbsLW38bf4MTh+REuDoBkYnuaZ5jM2K5517ZjE4yUykQXjzzpkD6jQTLHSSa1onWQlm3v/OGTx+ljngQ2e9RSe5pnURYTRgCpE28N7QSa5pYU4nuaaFOZ3kmhbmdJJrWpjz6ZTM3iQiNiA4p2uFVKA60EGcgI6tf4I5tjylVFxvdw6lvuvFfZmG1p9EZL2Ore90bP0jIuv7sr8urmtamNNJrmlhLpSS/IVAB9ADHVv/6Nj6p0+xhUzFm6Zp/RNKV3ItzIiIQUSmicjEQMcSKqQfU9KERJKLyL0i8i0R+amIeH8FuX4SkUQReVBEbhSR10RkdKBj6kpE4kXko0DH0ZWIZAI/AA4rpb4KdDztRMQoIt8XkZtE5Hci0vuZHn0Xk8HzP/YuMMvzXI6IPCwi80Tkup5eH/RJLiJjgAyl1F+AxcD9AQ6ps7HAUqXU68Cf8SyzHCw8Z/0rgapAx9KZJ65HgN8ppfYFOp4uCgCTUuo14K/A5QGOB6WUy/M/tgn3eoIADwPPKqUWAueISOKJXh/0SQ6cCXwJoJQqBqYENpyjlFJrlFJbPD+m4P4jBJOLcS8meeJZCgOjEGgB5orIYyISTNOu7ADOFJFsYDLwQYDjOZEMpVS953EJMOlEO4ZCkqcC9Z1+Dp4lND1EJAb3yejNQMfSTkTGAbVKqYpAx9KNScBGpdQC4C3gdwGOp4NSyg48BrwOjAD2BDaiE+q86FodkHaiHUMhyauAROgo5h2/tlEAeWK6H/i5UsoR6Hg6+SYwS0QeAgpE5CHPySgYtHZ6vBkYEqhAuhKRscAEpdQc4BPgiQCHdCKd8yAJqDzRjqGQ5KuA9u6F44A+denzg1uBRUqpmkAH0plS6gml1ONKqceBLZ7HTYGOy2MVR2+70unhHzQAzsddZAdYRhCdgLqoEJH2ualy6eFWMej7riulSkTkoIjMBYYDzwQ2oqNE5ArgeuAMT8uGQyl1R2CjCn5KqU0isl9E5gFZwI8CHVMnbwA/8VT4JgMLAhxP++3gN4CpQLSIROK+pbhfRMqB5Z3uz49/ve4Mo2nhLRSK65qmDYBOck0LczrJNS3M6STXtDCnk1zTwpxOcs1rRKRQRIK188gpSye55jVKqZUE0dgCzU0nuaaFuaDv8ab5hojcB9QCw4BPcQ+pNOIe8ZcL7FdK/UlELMB9wC4gA1inlFonImm4h9aWACOBF5VSRzzHvgD3CK5LgAt76o2l+Z5O8lOQiHwNMCulnvUMsHkVeBk4Sym1yLPPWyLyFvAA8DelVJFn3/dE5FLg18CPlVJVnmGZmcAR3KXDr5RSH4uIC5gJfOzvz6gdpZP81NTeB/oy3Ffv9okbOo8734X7Kj8ReBRAKaVEpBlIAIYppao8z5cD5e3H6DS8tYoehkBq/qHvyU9N2wGbUupdpdTbeJKYo7OOgLsIvhvYAoyGjmG1Ztzjl6tFJLd9ZxGJ90PcWj/oASqnIE+y/gL3xANluBMZ4E5gBe7RftuVUq96kvc+3AmfAaxVSq3xzH32v0ApUAP8HRgKvAJ8B/fQx18BbcCPlFI2/3w6rSud5BrgbuMGzlBK/TLQsWjepYvrGiISi3sW0HEiMjLQ8Wjepa/kmhbm9JVc08KcTnJNC3M6yTUtzOkk17Qwp5Nc08KcTnJNC3P/D5c1iOvAGPUIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPYAAAC4CAYAAADQQBHUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJztnXl8XNV597/PaGY0Wkb7bkneJS94V8wONkkamoQESpuFhBYICWQtJC/QNCFLKXkhW3mTZikESmkTwGVL0gRCQuwUwiJLXgHviyRbm2VZ0oxGo9nO+8cdybKtXTNz74zO9/OZj2bu3OUn6f7uOffc8zyPKKXQaDSphc1sARqNJvZoY2s0KYg2tkaTgmhjazQpiDa2RpOCaGNrNCmINrZGk4JoY2s0KYg2tkaTgmhjazQpiN1sAeORl5enFi1aZLaMUenv7ycrK8tsGaOitU0PK2trbGzsUkoVT3Z9Sxu7tLSUhoYGs2WMypYtW9iwYYPZMkZFa5seVtYmIk1TWV93xTWaFCSuxhYRm4jUiciqeB5Ho9GcSdy64iJSBtwAPK6UahKRtcA7gX7ggFLq9/E6tkYz24mLsUVEgK8BX1RK+aOL7wI+qpSKiMjjIvJHpVQ4HsfXaMzEt9/Hm1e/Sd7GPMo/UY57rTvhGuLVFd8ABIEbROQeESkFHEqpSPT7bqAqTsfWaEzl5G9P4tvjo+3hNhrXNeLb7wMgkUlN4tUVXw1sV0o9Gr2/fgJoGfF9L1AMHD17QxH5FPApgOLiYrZs2RIniTPD6/VqbdNgVmj7DVAE6hEFW6G+tR5agfuBAPBeYA1xHeGKl7FDI97vAtKBkf2RfKBjtA2VUg8CDwLU1tYqqz5+sPKjEa1tesRKW+uBVoJXBJl71Vy46vTyQ785RNvDbYT+GMI1z0XZjWWU3VCGq9o142OeTbyuGS8Da6PvS4A2ICAiadFl+cDxOB1bozGVik9WMPfLc89ZvvA7C7mw9UKW/mIproUujn79KK0/aQVAhRWRwcg520yXuLTYSqkdItIsIjcC5cCXgSzgLhHxAj/TA2eaVCTYHQQFjkLHqN+nudIo/WgppR8tZeDoADaX0bZ2v9jNnuv3UHZ9GWWfKCP7vOwZ6Yjb4y6l1HdHWbw9XsfTaKxA28NtHL7zMBefvBhHwejmHiJjXsbwe2epk/wr8jn+o+Mce+AY7vVuym8qp+zGMmzOqXes9cyzadD/dj/4J15PM/vwNHhwzXNNaOqzca91s3zTci5svZCF/7KQiC9C071NSJpMS4el54pbkbAvTGNdI2wErjRbjcZqeBo8uOum/9zaWeSk6rYqKv++kkBHYNrG1i32FPFu9xIZiMCLMHh80Gw5GgsRPBnEf9iP+x0zn5AiIqSXpU97e23sKeJp9BhvItC5qdNcMRpLMXRuzKTFjhXa2FNkzufncP7h8+FhqLyt0mw5GguRtSKLmodqLGFsfY89RUSEjPkZ0GS8j4Qi2Oz6+qiB9PJ0Km6uMFsGoFvsKRHoCrD3xr14thtdrtYHW6mvrSfs14/kNdD5ZCf+Zms8LtHGngKeeg/tj7YT7jOMnLEoA/9hPx3/MersWM0sYrB9kLc/8jYnnj5hthRAG3tK9NX3gQ2y1xmzgvI25uGuc9Py3RZUWJcjns14GqwzcAba2FPCU+8ha3kW9mxjaEJEqLqrioGDA5x41hpXao05eBo8xkV/zcymgsYKbexJopSir74P9/ozr8jF1xSTsTiDlm+3jLGlZjbgafCQuTRz+KJvNtZQkQQETwZxFjvJuSDnjOWSJtQ8WIOz2GmSMo3ZKKXwbPVQ8JcFZksZRht7kjiLnKzfs37ULBj5G/JNUKSxCiJC3Y66mIZdzhTdFZ8iRjq3c/Ef8/PWh98afhSmmV2kl6efEa1lNtrYk2T3B3dz6K5DY36flp1G9/Pd+l57FtL5ZCctD1jr/66NPQkiwQinXjyFCo39SMuR56Dilgo6N3UycHgggeo0ZtP2SJvl5jJoY0+C/t39RPwRcs7PGXe9ytsqkTSh5XvWunpr4odSygjVjEFEVyyJZ8GAZzDSDAM0Am+QpAUD+ur7AM551HU26XPSKb2+lPZH2pn39Xk4S/RIearjP+In1B2yzMSUIeI5Kv4rpdSjQx9E5EmStGCA5w0PjmIHrrkTZ5OsuqMKR7Fj2gHymuTCs9VaM86GiKexl4vI7RhJDH/I6AUDjsbx+DEja2UWzjLnmCPiZ6y7JIuF9y1MgCqNFRg8PkhadhpZ51mr/K7EuzqBiKwBPgOkK6X+NrrsW8CzSqmto6w/smDAuk2bNsVV33Txer1kZ48xfVBh3HgMApcnUFSUcbWZTEpqCxH3GSEbN25sVErVTXoDpVTcX8AfMYw89PknQPVE29XU1CizCfYFVWggdM7yzZs3j7vdtsu3qVcrX1XhwXCclI3NRNrMRGubHkCDmoLn4jIqLiIbRWR59H0+RoGTpCwY0PrjVl7JfYVQX2jilUdQfVc1g8cG6Xxcp09KVXz7fWzfsJ2+rX1mSzmHeHUgtgM3ich5wBLgH4FCkrBgQF99H+lV6dhzpvanKriygKwVWTR/u5nS60sRmx5MSzX66vvo/VMvtgzrPTWOVyWQHuD7Zy1uJgkLBnjqPeRemjvl7USEqjur2Hv9Xk7+5iRFVxXFQZ3GTDwNHmyZNjKXZJot5Rysd6mxEIOtgwweG5xwYspYlHy4hNxLci0VHKCJHZ4GD+61bkvmvNPRXeMw2YkpY2Fz2Fjz8ppYStJYhEgognebl4pbrJG88Gysd6mxENkrsllw/wKyV8/s8UwkEKH7xe6JV9QkDaGeEHkb8si9bOq3aYlAG3scMhZmUH1nNWkZaROvPA4t329h13t2GTW/NCmBs8jJyt+upPiaYrOljIo29hioiKLr111GWdQZUn5zObYMGy3f0cEhqUIkZO1xE23sMfDt9/HmB96k61ddM96Xs8hJ+c3ldPy8A/8xa+Sd1syM7ZdsZ8/f7TFbxphoY4+Bp96Y3J+zfnoj4mdT+cVKVERx7F+OxWR/GvOIBCJ4t3txllo3ek8bewz63ugjzZ1GZm1snlFmzMug5CMleOo9o+ZN0yQP/W/2owLKchFdI9GPu8bAU28Ez8cy/LLmpzWkZaVNKkpMY12GiwNYLLnCSHSLPQphfxjvTm/MuuFD2LPtiAjBnqCu95XEeLZ6sBfYcc2bOD7fLHSLPQo2p426nXWkZc7sMddoDBwdoGFlAwvuX8CcT8+J+f418Sf/Pflk1GZYuueljT0KYhOylsYncN4110Xm0kxavttC+SfLLTkdUTM+JX9dYraECdFn1Si0PdJG56b4hFuKCNV3VeM/7Kfr6Zk/StMklkBXgIFDA5YfANXGHoXm+5rjGkdddHURGbUZNN/fbPkTRHMmnU908saiNxg8Nmi2lHHRxj6LYHeQgQMD0w78mAxiE6rvqMa73YunUVcOSSY8DR4cpQ7SK9PNljIu+h77LIayTk43VHOylH68lOx12bhXW/eRieZcPFs9uOvclh44gzi32CKSIyIvRN//hYh8TkS+FE1waEn66vtAwL0uvoazpduGTa2748lByBvCt8dHzjvie9GPBfEsGCDAtUCniNiBG5RS14mIDfgF8JF4HXsm+Jv8ZC7JxJ6bmM7Mgc8fINgdZNnPlyXkeJrp493mBWW9HOKjEc8W+yrgeSACzAc6AZSRWzwzanbLseRnS1jXsC5hx7Nl2eh8ohN/kw4OsTpZ52Wx/Knl5Fw0S1tsEVkG9Cil2qP3IkXAyFSOXoxMpSdG2XZkXnG2bNkSD4kzxuv1xkZb9Kbk9btfh5tmvjuIobY4kPTaCoGdiVAzQ6aSq3iyL+BO4B+ir0bgK8CDI77/FZA20X4SnVe86zddavfVu9Vg++CE68YyB/Wu9+9Sfy77swoHYpOD3Mr5sZNZW+vDrcr7tjcxYs4CK+QVV0p9Wyl1n1LqPmA3cD+QAxDNLd6vLJh+uGdzDyefP4k9P7F3CRW3VhBoD3DyVycTelzN5AmeCrLvE/voejY5JhUl5AxWSoVE5GERuQ1wAfcl4rhTpa++D/caNzZnYh/vF1xZwLx75iXFoMxsxbvNCyTHwBkkwNhKqRuiP38PWLZ0biQUwdPgofzm8oQfW9KEeV+dl/DjaibPcKhmkhhbzzyL4nvbR8QXiXmo5lTo/l037f/RbtrxNWPTt7UP1wIXjgKH2VImxbSMLSLrYy3EbML9YXIuyCHnAvOM3fpQKwe/dFDHalsQ7zavpRMrnM2YXXER+QxwDUZR2DO+AsqAFXHUlXByL8xl7WtrTdVQcWsFXU930fV0F6UfKzVVi+ZM6nbVEe5LngvuePfYXqXUu0f7QkSuiZMe01BhFdM0SNMh/4p8MhZl0PrTVm1si2HPtmPPtuScqlEZsyuulHps5GcRyReR1SKSo5R6Nv7SEke4P8wrea/Q+mCrqTrEJlTcWkHvK714d3tN1aI5TcfjHRy5+0hSzemf1D12dDbYgxjzux8WkY/HVVWC8WzzEPaGcVaYn0629O9KyVicweBxa8f7ziY6n+jkxFMnLB/RNZLJ9i2qlVJ/M/RBRL4VJz2m0PeGMdvVzBHxIZxFTtbvW59UJ1Gq42nwkH9FvtkypsRkR8X3n/X5CICIzI+tHHPw1HtwzXPhLDG/xQYjfVIkEGHg6IDZUmY9g62DBFoDSfP8eojJtth1IlKNEallA5aJSDGwDEj6bnlffV/cEytMlV3v3UW4N8y6rYmLNNOcSzLkEB+NyRo7DPzviM8vR38mz2jCGKiIouKTFWQuj03Fj1hR/FfFHPjsAfoa+sips9ZFZzYRPBnEXmifcSnlRDNZY/878I9AAdAO3K2UahKRrXFTliDEJsz9ylyzZZxD6cdLOXTnIdr+rU0b20TKbyyn7IaypBvzmOw99q3A55VSfwHcgRGWiVLKFy9hiWLg0EBMSuXGGnuOndLrSun4RQeh3pDZcmY1yWZqmLyxdyqlTgAopTqAfQAiYu1UjZNg/2f3s2PjDrNljErFLRVEfBFOPHNOPorkpRlOPHuCSMDa9aUB/M1+tq7ayqnNp8yWMmUm2xXvE5FKjMEzAUIiUgG8C3hs3C0tjFIKT72H4muLzZYyKu51bta8tsZyA3vTQUUUYhN4Ht564i3sBXZKPlJC2d+W4V5vzayfnq0e+nf1k5Yd+1JP8Wayxv4M8G4MUw/xDqCaJDb2wMEBQqdCuM+37ohn7gW5ZkuYMcFTQbZfsp0F9y+Am2HF362g47EO2h9pp/XHrbjf4WbtG2stZ+6+rX2IQ8hemVwDZzB5Y9+ulGo4e6GILI6xnoQS6+L28eLI144Q7A5S8681ZkuZFke+egTfXp+RZL8HCt9ZSOGVhYR6Q5x4+gShnhAiglKKfTftI/eyXIqvLcaeY+7cbE+Dh6wVWdjSky+6eVKKRzN1dPmB0ZaLiFtEviIi14nIgyKySETWisgdIvIZERk1uCTR9NX3Ycu0kbnMWo+6zibUE6LtoTYCXQGzpUwZT6OH1p+0Muezc84pjmDPtVN+UzlVX6wCIHgiSO+fe9l30z5eLXuVt697m5MvnCQSSvz9uFIKT4Mn6Z5fDxGvS1Ex8L9KqV9gdNVvAe4CvqeU+jFwUzT3manM+ewclv1imeUrXlbcUoEKKNofTa4kDCqs2P/p/ThKHMy/Z+JJis4SYzrt2tfXUnZjGd0vdLP7L3cP5xlLpMHD3jCF7yuk4N0FCTtmLIlXMsPDSqmhSSzzgVcAhzJyigN0A1XxOPZUyKzJpOiDRWbLmJCs5VnkXppL609bUZHkmRPU/WI3nq0eFn1v0aQLMIgIOefnUPOjGi5qu4jlzyyn8P2FALR8p4Wtq7bS8r0WBtviGyRjd9tZ9vNllh1YnQiJVyiaiJRhpB/2AT8B7lVK/W30u28BzyqlzpngclZe8XWbNm2Kiz7agV3ARcA0xka8Xi/Z2QkcVPkDcC/wHaBu/FUTrm08dgPnMTzsOiNtfwT+G9iL0SSdjzGrIkbxGWdoG8BIu2mR8byNGzc2KqUm+M+PYCq5iqfzAuYC/4Jh5KFlP8GIGDMtr/ixHx9Tm9msBo4OTGv7ROfHDvvDat9n9k0qr7UVcncPdo6emz0W2rx7vOrQlw+pP2X8Sb31kbdmvL8hRmrbduk2tfO9O2O275mCFfKKRwfKKqMfPcAqIDDivjofOB6PY0+Wvjf6cJQ4SK9Ojjk2tnQbNT+qIWtpltlSJqT31V5eq3qN7t91x2X/WUuyWPCtBax8YSWLfrAo5vtXYYVnm4eMBRkx33eiiNfzhOPAx0XkJLAOY575IHCXiHiBnymTCwZ46j3knJ9juWenE+Fp9BBoD1D4vkKzpYxKJBRh/6f34yx2knNxfB8j5l2WZxwzEKH5vmaq/k8VaZkzH5P17fMR6Y8k7Yg4xMnYyph2+r3ox0dHfLU9HsebKqHeEL69PkquKzFbypQ5/OXD+Pb6uODIBabnaBuN1h+10r+rn+VPL09YjrDeV3s5+s2j9P65lxW/WjHj587JlkN8NKz9nCdOeHcY5VCtPjFlNCpurWCwZZCTz1uvHNBg2yBH7j5CwZUFFF2TuKcN+RvyqX24llMvnuKtD71FJDizx2KerR5sWTYya609v2E8kiftYgzJuzyPC49fmPAaXbGg8KpCnOVOWn/aStH7rfWormdzDyqiWPTDRQm/xSm/oZyIL8KBzx5gz/V7WPbzZdPu0RR+oJCMmgxL9ogmS/Kd2TEivSI5Bs3OxuawUX5zOU3/3IS/yY9rrstsScOUXldKwXsKcBSaUy1jzmfmEPaFafrnJgYODky7xS14d0HSTkwZYlZ2xfd9ah8nf2u9ruxkKb+5HHueHe8ua6QojgQi9LzSA2CaqYeo/j/VrN+7ftqmDpwI4Gn0zLg7bzazztiDxwdpe6iNgYPJmyjQVe3iovaLKLrKGl3xlu+3sOPSHXi2e8yWAkB6mdEba76/mUN3HJpSPvCuX3bRWNeIv8kfL3kJYdYZu6/eSDXsXp+8I54ANqcNpZTp2V/8zX6a7mmi6Ooi3Gus8zdVSuFv8dPy3RaOfvPopLfzNHhIy00jY2HyPsOGWXiP7an3GDG2SZacbjR2v3c3kWCE1X9YbZqGg7cdBAWLHoj9RJGZICIs/sFiIv0Rmr7ZRFpWGtV3VE+4nafBg7vOmokfpsLsa7Hf6CN7VTZpLtODy2ZM7qW59LzUg2+/OannTj5/kq5nu5h791xLDeINITah9me1FH+4mMN3Hub4jyaY7BiA/l39Sf38eohZZ2wUcZ8RlSjKbipD7GJazbFQt5F9pupLpgfqjYmkCUv/cylF1xQhjgla4cOggoqcdyT/+THruuKrN69OquJq45Felk7RNUW0/3s78++ZT1pGYnshpR8rpeS6Est3W20OG8ufXj6sM9gdHL2A/VxY9YdVZK9J/tu02ddik5zpZMei4tYKQt0hup7pStgxBw4N0PZoG0qppPlbDunsfa2X1+e/TtcvR/l7ZUD+O/NHN32SMauMfejOQ+z+4G6zZcSUvI15LH92OcV/k5iEAEopDnzuAAe/cJBgp/XysU9E1nlZZC7N5K0PvXVu9Nlzp5+aJDuzytin/niKcL+pQWUxR0QovroYmzMx/8quZ7vofqGbef80D2epNYoYTgW7287K51eStSyLN695k54/GRNrwr4w/BBO/k/yTlwayawxdtgfpn9nf1IGfkyGlu+3cPSeo3E9Rrg/zMHbDpK1Mos5n5sT12PFE0e+g5UvrsQ1z8Xu9+/Gt89nBAZFkjuiaySzxtje7V5USCX9xJSx8O700vKdFkLe+JUDOnrPUQZbBqn5cY3lE0BOhLPYyao/rKLytkoyFmWkRKjmSGbNqPhwDvEUqKoxGhW3VtDxWAedj3dCnLK9512ah9iF3IuTv4gBGIFAQ9lThzKhJmtw0NnExdgikgd8AugA3gPcg5Ey8J1AP3BAKfX7eBx7LNIr0ym5roT08tT4x51NzgU5ZK3MovUnradTXMSYwvcVWjZzy0xQSuHb54PLzVYSO+LVn1oKvKiU+i/gEYyso1PPK+4z0tSEB2Y+4FV8bTHLfr5sxvuxKiJCxa0Vxr3iqGUcpk/npk6OfP1IUhTSmw4iwvo9642cuilCvFIjvTbiYyFGEtoF6ty84kfH3dExqF9SD4Cj2IFrrouah2pwr3YzcGgA7y4vrrku0qvTcRQ6xnymGhmMEBmMmF4yJt6UfqyUtofb8M4zwjn33rjXiNle4CJjfgauBS4yl2aeU5FjPEK9IQ7+/UHSK9OZ97V5cVJuPvZcu5FuOEWI65kuIpnApcD3MbrhQ/RiVAs5Oso2w3nFy/PKjXKAHRDsCBLsCNL4diP0AE8D/zpiQxdQAnwbKMUo9NsUfd8WXf5DYHlsfjev18uWLVtis7NY8nXwBqLaBoBOYCfGpRRgNUYyaIC7Meqnlo94zQMqRuzvX4EOCHwjwJ9e/tOM5Vn274a1tU2VuBlbjObzduCbgBcY2UzkY9x/n4NS6kHgQYDa2lq14d4No+4/tDbEwI0D+Jv8DDYP4m/y42/2s+R9S7Bn2zn0m0O0fLfl9AY2uOT6S2LWam/ZsoUNG0bXZjbD2jacXhb2hfEf9aOCiuxVxpTJtyvfpv+tfgZ2DhDpNzpTpdeXsvSxpSil2PmunfRs6aHilgpqbolNQcCk+LulAPFssT8BPKqU6gYQkYCIpEXTDs84r7g9x457rRv32tG7lfP+aR7lnywfNr690J7yXfHxSMtMI2vZmTnJlz1ujDkopQieCOI/4seWaQy7RAYMo+deksv8eyeuu6WxFvEaFf8r4KPAJdH73kHgPhKYVzwtI43Mmkwya5I302SiEBGcJU6cJadnkqVlprH6JfPivDUzI16DZ88Az4zylSXyims0qU5yTx/SaDSjoo2t0aQg2tgaTQqija3RpCDa2BpNCqKNrdGkINrYGk0Koo2t0aQg2tgaTQqija3RpCDa2BpNCjJ7w51MJBSOsL/Dy65jPew81sMJzyAr5uRRNy+fVVV5ZKenxr/FFwixv8PL/nYPe9s9HOj00HVygCePNZKX6SQv00F+poO8DON9XqaT/EwHudFlziRPmGgmqXEGWRilFM3dPnYe62VnSw+7jvWw+3gv/mhh9RyXnSJ3Oi/t7UQpsAksLc9h3dz84decvAxLV9wIhSMc6epnb7uH/R2Gife1e2g55WOompLLYWNxiZuwggOdXnp8QXp8AUKRscstZTnThi8AQ8bPy3CQH11WkuPi8ppicjOsUblDKcXrh7vZ1NDCS3s6UArS0gS7TUizCXabLfrT+JxmExxpZy6zpwlpNtuIbYyfU0UbO8ac8AwOG3jHsV52Heuhx2dUzEi321hekcNH11ezqjKPVVV5zCvMREToHQiyo6WHxqPdNDaf4qnGYzz2WhMApTnp1M0tYO3cfOrm5rOsIgdHWuJbM6UUrb1+9rX3sa/dy772Pva2ezh8op9A2LhQ2QTmF2WxYk4u166tpLbMTW2Zm+qCTNJsEk1mcPnw/voDYU71B+gdCHLKFxg2fI8vyClfkJ6B08vaevuG3w9dD5x2G+9cUsLVa+awobaYdHviq6h29Pl5qvEYmxpaaDrpw+2yc+XyMnIyHIQjimA4QjiiCEXUiJ8RQmHjc3DE58FghFAkfM56U0UbewZ4/EF2H+9lV7Q13tnSQ2uvHzBO8JpSN+9ZVsaqqjxWVuZSW+Ye05C5GQ4urynm8hqjVE8oHGFvu4dtzadoOHqKxqZT/GZ3G2C0fisr86gb0arnZU6/KkckougPhPD4Q3gHR/4MctIbYF+H0QLvb/fgGTydt7w810VtmZvLa4upLTUMvLA4G5djcuYSEbLT7WSn25lKvc5IROEZDHH4hJdf7Wzl1ztbef7NdnIzHLx3RTnXrJlD3dx8bNNo6SZLMBxh895OntzawuZ9nUQUXLCggNvetZi/PK980n+DySJ3THH9RFSeFBFR0zhQevlitfTTP6Ig00l+lpP8TCcFWQ7ys5wUZDopyDJeQ5/zs5zkuOzT6raGwpHhk7rPH8TjHzrBT78fWt47EGT7oXbafGq4q1ldkMmqqjxWVeayqiqP5RU5ZDpje91s7/XT2HQq+urmrda+4a7swuKsaKueR/Oh/VQvrDnTqNH3ff4g3sHTn4fWGY8cl50lZTnDrW9tmZuaUve0usDxSD8UCkd45WAXv9zRygtvtjMQDDMnL4MPrq7gmjVzWFw6ueSNk9F2+ISXTQ3HeKrxGF3eQUrc6fz1uko+VFfFvKKscbedCSLSqJSqm/T68TK2iNiA64C/Br6jlPqziMwHPoaRFmlAKfXEePsonVejPvXAM3T7ApzqD9DdH+CUz/gZHKN7YrcJeUMXgEwnhdnGBcHlSDvj5DbMetq0A8GJE7o402y4XXbcLju5tkGuWLWQVVW5rKzMoyAr8XWsBgJhdh3roaHpFNuaTtHYfGq42z+SLGcabpeDbJfROg79DkZraSzPGfp8xjoO8jIcFLvTY3aPH++8Yv2DIX7/dgfP7TjOywe6CEcUyytyuHr1HD6wuoLSnLFTkY6lbSAQ5re723hyawv1R7tJswlXLCnhw3VVbKgtxp6A2yLLGHv4ACLfAP6glHpFRB4GbldK9YnIQ8AdSqmesbatra1V+/btO2f5yHuz7v4A3b4A3d7Tph/+2R8cvigMBMNnnLBDJ7c73XHGsqGT/PQ6juHtRnavrJj4TinFka5+Xn29nisuvZBsl50sp31agy/xIpF/txOeQf5nVyvP7WhlZ0sPInDxwiI+uLqCK88rw+06s8cxUptSit3He3liawu/3tGKZzDE/KIsPlRXxbXr5lDiTmyu4qkaO9H32KVKqaE6pfsxkuFumepOzrg3K9A5zYYQERYUZ9PstlGRl2G2HNMpdqdz48XzufHi+Rw+4eW5Ha38csdx7nhqF1997k3evayUa9bM4bKa4uGxjx5fgOe2H+eJrS3sbffgcth474pyPlxXxfr5BZZ+OjGSRBt7ZH91KLe4RhN3FhRn88V313D7uxazvaWH57Z4LL6CAAAFRklEQVQf59c7W/mfXW3kZzp438pyDjb52faHlwiEIqyszOXea87jqlUV5Lis8ThtKiTa2IER7/OBvWevMLJgQHFxsWUTuFs5ubzWNjFX5MJllzh4s8vGa60hNtU3Y7cpLp/j4LLKdKrcIRg4wrbXj5gtdXoopeL6Ar4BXBJ9/zMgd8T7nPG2rampUVZl8+bNZksYE61t6gwEQur3L/3RbBljAjSoKfgunpVAMoH3AesAl4g4Mapu3i4ixzEG1PrG24dGkyhcjjTsFhpknClxM7ZSygf8d/Q1km/E65gajcZAz7LXaFIQbWyNJgXRxtZoUpCEzBWfLiLiwah0bUWKgC6zRYyB1jY9rKytVik1uUnvWD+6a5+awjS6RCIiDVrb1NHapoeINExlfd0V12hSEG1sjSYFsbqxHzRbwDhobdNDa5seU9Jm6cEzjUYzPazeYmtSDBGxiUidiKwyW0syINOME7WssUXkcyLyMRG5W0QKzNYzhIjkiciXROTjIvKfIlJjtqaRiEiOiLxgto7REJEy4E7ghFJqp9l6hhCRNBG5Q0SuF5EHRKTaZD226Pn1HHBRdNl8EfmqiNwoIh+ZaB+WNLaILMFIyvBzYBNwu8mSRrIUeFEp9V/AI0RDTK1A9Op+LdBptpaziWr7GvCAUqrJbD1nsQJwKKX+E/gFcI2ZYpRSkej5tQMYarG/CvxAKfXvwDtFJG+8fVjS2MClQCOAUmofsNZcOadRSr2mlNod/ViI8ce3ClcBzwMRs4WMwgYgCNwgIveISKnJekayF7hUROYAa4DfmqxnNEbLPjQmVjV2ETAypNNyKSyiYamXcm70mimIyDKgRynVbraWMVgNbFdK/RR4CnjAZD3DKKX8GCHF/wUsAKyYXWFK2YesauxOIA+Gu3CB8VdPLFFNtwPfVEoNmq0nyvuBi0TkH4AVIvIP0YuPVRiZ43gXTCmVeFwRkaXASqXURuAl4NsmSxqNs7MPdYy3slWN/TIwNLVvGTCl6XQJ4BPAo0qpbrOFDKGU+rZS6j6l1H3A7uh7n9m6RvAyp2+pSpjgxEwwf8HpNF2/x0IXnRG0i0hu9P1iJrgFtORccaXUfhFpFZEbgHnA/zNX0WlE5K+AjwKXRJ9EDCqlbjFXlfVRSu0QkWYRuREoB75stqYRPAF8JTpoWwD81Ewxscg+pCeoaDQpiFW74hqNZgZoY2s0KYg2tkaTgmhjazQpiDa2RpOCaGNrZoSIbBARK07omNVoY2tmhFJqCxaay68x0MbWaFIQS84808QHEfkC0APMBf4XIzwxDSOSbjHQrJT6NxHJBr4AHAJKgTeUUm+ISDFGmOp+YCHwkFLqZHTf78GIjPoAcKWuy2Yu2tizBBG5AshQSv0gGsTyGPAwcJlS6tHoOk+JyFPAF4HHlVJvRtf9pYh8EPi/wD8qpTqjIY5lwEmMnt9OpdTvRCQCXAj8LtG/o+Y02tizh6F5x1djtNJDyQ5Gxm4fwmjNVxEtnqiUUiIyAOQCc5VSndHlx4HjQ/sYES7ayQQhhZr4o++xZw97AI9S6jml1NOcrno6MqfWQuAwsBuogeEQ1QyMGOAuEVk8tLKI5CRAt2Ya6CCQWULUoP+EEbB/DMO8ALcCmzGi6PYopR6LGvYLGCYvBV5XSr0WzQX2LeAA0A08CVQD/wF8HiOU8F4gDHxZKeVJzG+nORtt7FmMiGwALlFK/bPZWjSxRXfFZykikoWRAXOZiCw0W48mtugWW6NJQXSLrdGkINrYGk0Koo2t0aQg2tgaTQqija3RpCDa2BpNCvL/AfRLXDJT+AewAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_sampling(net, train_iter, valid_iter, 0.001, 10, lang, device, first_train=False, lloss=2.426,lppl=11.588)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 1.898, PPL 6.741, 8.5 min on cuda:0\n",
      "loss 3.067, PPL 21.816 \n",
      "0.001\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC4CAYAAAAsVToeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAGY9JREFUeJzt3Xl0XPV5//H3M5tG+2ZZko1teTfesLFJsLEbOyGYJiEhlHMov0BKmqRtgKQ4NEtJIYQ0pGQpSSFL4YQlIbSQwC+Jk7AGFCA/yK8GAzYGrxgvsrxIlmytI819+se9MxrbkiWNPZqR7/M6R0d37jbP2Pp85947c79fUVWMMf4TyHYBxpjssPAb41MWfmN8ysJvjE9Z+I3xKQu/MT5l4TfGpyz8xviUhd8Yn7LwG+NToWwXcDLKysp02rRp2S6jX+3t7RQWFma7jH5ZbenJ5doAXn755YOqWjXU9Ud1+Kurq1m7dm22y+hXfX09K1asyHYZ/bLa0pPLtQGIyDvDWd8O+43xKQu/MT5l4TfGp0b1Ob8xI0FVeePSN+BPsP5d6yldVkrpslKKFxUTyBu9758WfmNSxPbFOPT0IZqfasZpd5jzizmICMGiIMyCjs0dNK1pAqDmqhpm3TsLVaX58WZKzi0hXB7O8isYOgu/McDee/ay+/u7aX+9HYBQRYiKVRWoKiLCmfefyb76fbx7xbuJHYjR+qdW8mrzAOjc2sn6D6wHoHBuIaXLSik5r4SKCyqIjI1k7TUNxsJvfEUdpW1dG81PNXPoqUPM/u/ZRKoiOJ0O4cowk2+dTPn7yyleWIwEpd99RKoiVF3c93F6dGKUBfULaH2hldYXWtn34D4aftzAnEfmUHVJFR2bOmh+qpnSZaUUzSsacL8jzcJvfKH9zXZ2fHUHh/5wiN7mXgAK5xfSvaebSFWE8deMZ/w149PadyAvQNl7yih7TxkAGlfaN7QTnRwF4NDTh9j62a0ABIuDlCwtoXRZKWdcdwahouxF0MJvTjtOt0NLfQsH1xyk/Pxyqi6uIhAJ0PqnViovqqTi/RWUva+MvJq8jDy/BIWis4qSj8ddPY7KD1UmjwxaX2hlx1d3MOH6CQBs+/I2Dj5ykPzp+cmfgukFlF9QjkjmjhIs/Oa0oKo03tdI05ommp90L9YFCgJE69x33/yp+SzZvSSjYRqIiBCdFCU6KUr1x6oB6G3tJZgfBNzrBF3bu+jc0knLcy047Q7hqjDn7T8PgK2f30rH5g4Kphf0NQ4zC4hOjJ5UXb4Pv6py6OlDOF0OYy4aA8D6i9fjdDlIUJCAQBAqVlUw/jPuYeHGKzYiAXHP3YJuS1/+/nLGXjoWp9th+w3bYR/s3rCbvHF5RMZFyJ+WT2RM7l78GW1U3UPrzi2dVF1ShYiw6zu76G3tpebKGiovqqRsZVkyYEBWgj+QUGlf9GquqKHmihrAfV2xxhixfbHkcgkL3bu6aXm2BafDAaBoYRGLX1kMwLYvbaP4nOLh13AyL2A0c3odDj5ykJ237aRtXRslS0uS4Y8fiRNvi4Pjnr9pXOnZ3wO4/zmHXzoM8b5lGlcitRG4FJyYw9679kIHbP351uTz1X2tjrqb6uhu7Ob1Va8nG4W8cXlEaiOUv6+cgpkF7v4cJRAevZ8fZ4rT7dDyxxaa1jTR9NsmunZ0ESwNUvnhSgKhAAueWUB4bDinQj5cIkJebV7ykwSAqbdNZeptU92GoSFGx5YOiPdtc/jPh8kbP/xTGF+G/8CvDrDt+m10be8if0Y+M+6eQc2VNcnlC/6wYMBtRYRzt5474PJQcYjlR5ZT/0w9S+cupbuhm1hDjOgU9xBNe5RoXZRYQ4y219uINcbAgZn3zKRgZgFHXjnCK+9+hfDY8FENxLh/GEfx2cU4MQcCEAj5o3GIHYgRKgsRCAfYccsOdt66k0B+gPLzy5l4w0QqP1iZ/LeIVJ/eR1YiQt74vOOCvrB+oTvxj8Pbn2/C39PcAwEIl7lfwghXhZn6namM+cgY99D+VAtAZGzE/Zw3pS2JTogy79fzko81rsT2xwgWuoenkbERJt00iVhDLNlwHPmfI4y9fCwATb9rYuNlG93zvlkFFJxZQMGsAio/WDmqvmAyIIW2DW00/baJpjVNHH7xMPOfnE/F+RXU/E0NJUtKKH9vOcGC4OD7Mid02oe/a1cXu2/fTcNdDUy4fgKTvzaZMR8Z44Y+Bw4PJShHHeJFJ0WZfPPk49ZLDKuWPzWfMz5/Bh1vddC+oZ2Dvz4IcXjXpncRLg/T+LNG9j2wL9koFJ5ZSMGsgpw6HFZVepp66H6nm64dXeRNyqNkcYl7nns5rN3n3qZdtKiIuq/WkT8tH4CCGQUUzCjIZumnlYyEX0SKgc8BbwMrgG+p6taU5ZXAtcBWoFRVf+jNvxY4BEwBfqCqzenW0LahjV3f3sX+B/ejqlRfXk3VpVWJ5093t1mTqLlofhFF8/s+RnJiDp1bO/tOK+JKz8Ee9t69N3lxCGDZkWWEikIceOQAPAF73tpDqCREqDREqDxE6dJSAOLtcSRPTuq0QtWtoWtHF107ughXhil/bzkaV9YuWEvn25047X21jf/seEoWlxAqD8FMmHHLDCo/WJnWeawZuky981cBz6nq8yKyE/h74Aspy68H7lXVLSLyDRGZDgSBalW9U0RmAquBG9MtYMeNO2h+splx14xjwuoJRCed3MciuSoQCVA4u693mdqraqm9qhZ1lO7d3XS82UHXO13JL5Mc+L8H4OewhS3JbSI1EZbuXQrAxss30rSmiUBBgFBJiGBJkMI5hcx9dC4AO7+9k+7d3YRK3WWhkhDhseHkN97WvWcdR9YeOarhGXPxGMrfW44EheLFxZSfX060LkrepDyidVHyJ+cnXwtfg3ErxmX2H80AGQq/qm4HtnsPJwPPHbPKfFVN/PW9CpwHhIGXve03icjZQ34+Rzn4m4Ps+s4uZt07i4LpBUy9fSozi2cSrjwNzoPTIAEhOjF63GfBsx+Yzf6P7WfJgiXED8fpPdyL9vSN1Fz98WqKzymmt7U3uTw8pu/fsPW5VlqebyF+OA7eZkWLipLhLzm3hOJFxUTrokf9JMy6d1YGX7UZDsnUEN0iUgN8GegAblLV3pRlT6rqBd70BcBZuA3Rn1X1mWPXOWa/fwf8HcDYqrGLHrrqIXgI2AnU4h5fLMzISxqWtrY2ioqKBl8xC05JbQ7QBbQDBcAp6trutP93y6CVK1e+rKqLh7p+xi74qWojcJ2ITAK+jXsYnxBLmS4H9uG+85cBiHuCm7pO6n7vAu4CmBmZqXwbihYUMeG2CVRdWpUzH4Hlcn9vVlt6crm2dGQkKSJytoic4T08ApwlIhERSXwQu05EEsd/C4EXgOeBRKs1Gxi8Z85ymP/EfBa9sojqv67OmeAbMxpk6p1/D3CFiDQBi4AbcK/+C+5RwO24RwXbgJ3eNQJEpEFErgLqgO8P+ixlUHFBRSbqN+a0l6kLfvuA73oP7/N+v5SyvBm4qZ/t7sxEPcaY49lxsjE+ZeE3xqcs/Mb4lIXfGJ+y8BvjUxZ+Y3zKwm+MT1n4jfEpC78xPmXhN8anLPzG+JSF3xifsvAb41MWfmN8ysJvjE9Z+I3xKQu/MT6VqUE7yoBP4nbMuQr4uqpuTln+PSDRDeoY4B5V/Y2IPAokBup4WVV/lIn6jDGZ68PvTOBJVV0vIntwu9r+p8RCVb0uMS0inwUe9x7+RlXvy1BNxpgUmerD78WUh5W4A3McR0QKgHxVTXTTPUdEVuP2An+HqrZmoj5jTAYH7YBkuL8JfFFVu/tZ/hng6ZTRexLzFwJXq+qn+9kmOWhHVVXVoocffjgjtZ+sXB7gwWpLTy7XBsMftANVzcgPbjfdXwEqBlgeBO46wfZPD/YcM2bM0Fz17LPPZruEAVlt6cnl2lRVgbU6jIxm8mr/J4H71Btp95hBOwA+DDyTeCAiK0VkjjddDjRmsDZjfC9TV/svAS4HlnlDS3cDW+gbtAPgSm+dhHXA34rIXGAW7kAfxpgMydQFv0eBRwdZ55JjHrcA/56Jeowxx7Mv+RjjUxZ+Y3zKwm+MT1n4jfEpC78xPmXhN8anLPzG+JSF3xifsvAb41MWfmN8ysJvjE8NKfwi8gkRmS0i80TkIRG5KsN1GWMybKjv/K2quhH4Eu6tutHMlWSMGQlDDX+piJwL7FTVNqApgzUZY0bAUMP/GnAhcJuILABqM1eSMWYkDOl+flV9BXgFQETagR9nsihjTOYNKfwicjPwMLAUmA20AzeeYP3B+u1fAXwGOOLNukNVXxORa4FDwBTgB4kuwIwxp95Qe/J5HXgLuEVVLxWRvxlk/RP22+/5kqruSDwQkVlAtareKSIzgdWcoIExxpycoYZ/MnAH8LCIFAHFJ1pZB++3vxv4P17/fjtU9UFgOfCyt/0mETl7iLUZY9Iw1PDfDkxU1R0iMg34n6Fs5PXbvxz4Yup8r3F40VvndhHZhTts17aU1cJDrM0Yk4YhDdoh7lv0x4GzcK/8/1QH2dDb5gbgRyc6dxeRVcB83DH6Dqnqo962a1T1Q/2sb4N2nCSrLT25XBtkaNAO4AvAJbhdal+MOwLPYNt8Chif8jgCRLzpa+hreK4GPgDMAG715s0Bbh7sOWzQjvRYbenJ5dpUhz9ox1AP+3ep2x03wFsi8rETrTyEfvt3ApeISAnuuHyPqaqKSIP31eE64PtDrM0Yk4ahhv/YY53CE62sg/Tbr6prBph/5xDrMcacpKGGf6+I3If7jj2BQQbkMMbkvqF+w+93IvICMA33ivzyjFZljMm4AcMvIrcBZwPHXtUXoAbo99DdGDM6nOid//eq+qX+FojIkgzVY4wZIQPe1aeqfzzBshcHWmaMGR2sGy9jfMrCb4xPWfiN8SkLvzE+ZeE3xqcs/Mb4lIXfGJ+y8BvjUxZ+Y3zKwm+MT1n4jfEpC78xPjXUzjyGZQiDdpwPVAETgfHAP3rdeD2K25EnwMuq+qNM1GeMyVD4GXzQjom4PQD3isj9uB12bgB+o6r3ZagmY0yKjIRfBxm0Q1XvSXkYAbZ703NEZDVuH4F3qGprJuozxgyx3/60d+4O2vFN3K6+u/tZ/h5gmqr+5Jj5C4GrVfXT/Wxj/fafJKstPblcG2So3/50fnC7+/oKUDHA8onAdSfY/unBnsP67U+P1ZaeXK5Ndfj99mfyav8ngfvUG61HRCIiEvGmK4APk9I3v4isFJE53nQ50JjB2ozxvUxd7R9s0I4fefMWe8v/gNsh6N+KyFzckYFuyERtxhhXpi74DTZox2UDLPr3TNRjjDmefcnHGJ+y8BvjUxZ+Y3zKwm+MT1n4jfEpC78xPmXhN8anLPzG+JSF3xifsvAb41MWfmN8ysJvjE9Z+I3xKQu/MT5l4TfGpyz8xviUhd8Yn8rWoB2VwLXAVqBUVX/ozb8WOARMAX6Q6P/PGHPqZeqdPzFoxwPAPXhdbae4HnhQVX8OjBeR6SIyC6j25j0MrM5QbcYYsjRoBzBfVbd4068C5wFh4GVv+00icnYmauvP1v1H+MXa3dSWRlk1t4ba0vyRempjsiZTw3UByUE7lgNfPGZRJGW6FajzatmWMj88wD5TB+2gvr4+rdpUlc2HHH7/dg+vHYgTEHAUbl6zkSmlARbXBFlcHWJsQXoHR21tbWnXlmlWW3pyubZ0ZCz84vbJvRr4mh4/Wk8sZboc99pAGChL2TZGP1T1LuAugJkzZ+qKFSuGVVdv3OHxNxq5+7ntvLa7lYrCCNedP4Urz51ES2cPj29o5PENjTy8qZWHN/Uwu7aEv5xbw4Vza5heXTzk56mvr2e4tY0Uqy09uVxbOjL5zn/coB0AqhoD1onILFV9C1iIG+YQcBVul9+zgbWnspj27l4eXruLn7zwNrsPdTJ5TCHf+Ohc/ursM4iGgwBUFuVxzcppXLNyGruaO3jijUYe29DId5/azHef2szUqkL+cm4tF86tYc64ErwxB4wZlbI1aMftwHUisg3Yqarbve0aROQq3NOA7x+/5+Hbf7iL+1/cwQMv7aS1s4fFk8q58UOzOf/MaoKBgcM7oaKATy2fwqeWT2Hf4S6eeMM9Ivhh/VbufHYrEyryuXBODRfOrWXhhDICJ9iXMbkoW4N2NAM39TP/zlNVw5Z9R7j7+e38al0DPY7Dqtk1fPovprBoUvmw91VdEuXjS+r4+JI6mttjPLXRPSK47//t4O7n36amJMqqOdVcOLeWc+rKCQXt6xMm92X0gt9IU1Ve2t7M3c9v55m39hMNB7jsnAl8ctlk6sYUnpLnqCiMcNk5E7nsnIm0dvbw7Fv7eWzDXh5au4v7X3yHysII759dTVVvL2MbDjOuLEppfthOEUzOOS3C3xt3+P0G9yLe+j2tVBZGWH3+DK5cMomKwsjgO0hTaX6YixeO5+KF4+mI9VK/6QCPbWhkzWsNtMfi3LHueQDyw0Fqy6KMK82ntjRKbVk+48ui1JbmM877XZh3WvxXmFFkVP/FOQr3vPA2P3nhbfa0dDJlTCG3fnQel5w9PnkRb6QUREJ8YF4tH5hXS1dPnJ//rp7aqbNpaOlkb2sXe1s7aWjp4o+bD3CgrRt3FPI+JdEQ48rcxmFcWX5yOtFAFEfD5IUCREIBQgGxIwlz0kZ1+Hcfcbjltxs5p66cr17kXsTLhQtv0XCQqWVBVsyr7Xd5rNdh3+GuZKOwp6WTvS19DcSru1o41NEz4P5FcBuCYIC8cND73fc4L+g2EonGou93kEgoQOOeGGu7NxEICKGAEAwIARGCAQgGAgQFd5633F2W8iMpy7zf4WCAcFAIBQJ900G3oUo0WKGgW2MoKNaA5YBRHf5oSHj06qWcPXH4F/GyKRIKMKGigAkVBQOu0xmL09DqNgoNrZ10dPfS3esQ63Xc3/HEdJzuxLzk7zgdsV4OdaTOc7fp7onT1RNHd2zF0QGffkQkGo1Q0PsdEOI9MQr+/zMExG10BLexSz5OTAfwHguB5HKOeeyuL95+EvOPnufuS+ibHxCSyyT5nNDY2M0Tza8f9RxufZJ8/kAgdV5fXfRTp0jK89JXGxxdc9+6fftJ3S5R97D//U/B/2HWjC2QURf8ocqPBJlaVcTUqqJTvu/El1VUFUeh13FwHIirEnf6fhxVeh3FScwbYHncUXp6HXocpTfu0BNXeuIOvU7KdHLe0ev2Okqs1123N67s2rOX6uoKHK82RxVVUBTHITnfrf3odZyUeXHHIe4oCu726k476u7HnX/0dqnrOinPmdi2syvOm637U9bpq6VvP6Ts6+j6cs2oDr85OSLiHeKP7PWRE6mvb2bFigXZLqNfJ/sNv9TGIO4ddiUaNvd3SkORMv+ohsldkGycEtupKhNuG149Fn5jRkiysUUY4evR/bJvoxjjUxZ+Y3zKwm+MT1n4jfEp0Vz8DGKIROQIsCnbdQxgDHAw20UMwGpLTy7XBjBTVYfc6cRov9q/SVUXZ7uI/ojIWqtt+Ky29InIsPrAsMN+Y3zKwm+MT4328N+V7QJOwGpLj9WWvmHVN6ov+Blj0jfa3/nNaUZEAiKyWETOynYto4WkeW/0qA2/iFwrIh8TkRtFpCLb9SSISJmIXC8iV4jIz0RkRrZrOpaIlIjI49mu41giUoM7xsMBVX0t2/UkiEhQRL4gIleKyPdEZGIO1BTw/sZ+BSz15k0WkX8RkU+IyF8Pto9RGf4cH9prsKHKssp7l/grYH+2a0nl1XUT8D1VfSfb9RxjHhBW1Z8BDwIfzXI9qKrj/Y29inerP/AvwH+o6r3A+7wxMwc0KsOPOwpQcmgvYMSG9hqMqr6oquu9h/0NVZZtFwGPAU62CznGCqAHuEpEvi4i1VmuJ9VbwHIRGY87zsTvs1zPQKpV9bA3vRk44b3RozX8Y4DDKY/7Hdorm1KGKvtFtmtJEJHZQIuqNma7ln4sANap6o+BXwLfy3I9SaraBXwdeAB3BOm3s1vRgI4dBq/qRCuP1vDvZwhDe2XLIEOVZdOHgKUi8mVgnoh82WukckFvyvTrwIRsFXIsETkTd3DZlcAfgG9luaSB9DcM3oBGa/ifBxJfszzlQ3udAkcNVZYrVPVbqvpvqvpvwHpvuiPbdXmep+/0bSyD/OGOsAtwD/0BniKHGqZjNIpIqTc9nUFOOUfld/tVdXMmhvY6FfobqkxV/z67VeU+VX1VRHaKyCeAWuCfs11Tiv8GvuJdaK4AfpzlehKnlR8EFgFRbyzMrwOrRWQP8HTK+X//+7Av+RjjT6P1sN8Yc5Is/Mb4lIXfGJ+y8BvjUxZ+Y3zKwm8yTkRWiEiufjHGtyz8JuNUtZ4cuv/CuCz8xvjUqPyGn8kcEfkc0AJMAp7DvX01iHsX5XRgp6r+p4gUAZ8DtgHVwJ9V9c8iUoV7G/NmYCpwt6o2eftehXtX3IeBCwf7BprJLAu/SRKR9wL5qvof3s1JPwV+AvyFqt7nrfNLEfkl8Hngv1R1g7fur0XkI8A3gRtUdb93C2wN0IR7lPmaqj4hIg6wBHhipF+j6WPhN6kS3xO/GPfdPtGpRuq9/9twjwrOAm4GUFUVkU6gFJikqvu9+XuAPYl9pNxKvJ9Bbjc1mWfn/CbVm8ARVf2Vqj6CF276eooB91B+O7AemAHJW5jzce8hPygi0xMri0jJCNRt0mA39pgkL8S34HYKsRs34AD/ADyLewflm6r6Uy/Un8NtCKqBl1T1Ra9/u1uBLUAz8BAwEbgf+CzubabfAOLAP6vqkZF5deZYFn5zQiKyAlimqv+a7VrMqWWH/WZAIlKI2zPsbBGZmu16zKll7/zG+JS98xvjUxZ+Y3zKwm+MT1n4jfEpC78xPmXhN8an/hd0NkF8wC1CogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPYAAAC4CAYAAADQQBHUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAFuxJREFUeJzt3Xt4XHWdx/H3dy65TO5J0zRpLm3TJm2hUKQFYWE3yEX2QVfRVUBdFmQXXRRWvILibXFVWHUVFRV0RaxXEHEfnkVAJOtlabEVSgtN0/slbe5pk5k0t5nv/nEmk0lN2twmMzn5vp6nz5w553fO+c40n3N+Z845M6KqGGPcxZPsAowxM8+CbYwLWbCNcSELtjEuZME2xoUs2Ma4kAXbGBeyYBvjQhZsY1zIgm2MC/mSXcCp5Ofn6/Lly5NdxphCoRBZWVnJLmNMVtvUpHJtW7ZsaVfV4om2T+lgl5SUsHnz5mSXMab6+nrq6uqSXcaYrLapSeXaROTAZNpbV9wYF7JgG+NCFmxjXMiCbeY1VUUj0e8kaIXWn7cSGYwkt6gZYME289bxjcfZeulWmr7e5Ix4Gl695lU2Vm1k/2f303+0P7kFToMF28w7we1Btr15Gy9e8CKh7SG8uV5nwnWw5ok1ZJ+dzf7P7Gdj5UZ2XL+DufgtQyl9usuYmXbg8wfYd9c+vDlelty9hPIPlOPLjsbAC0VXFVF0VRG9u3s58q0jhENhRASA9l+1U3BZAd4sbxJfwcRYsI3r9R/tx5PuwV/oJ/eCXCo+XEHlxyrxF/nHnSewPMDyL49cHNXb2Mv2N2/Hm+el9MZSym4pI7AiMBvlT4l1xY1rDXYNsueOPWyq3sSBzznXdxRcUkD1vdWnDPVYMldkcs4fzqHob4to+kYTL9S8wNbXb+XEnhOJKH3abI9tXCccCnP4a4c5eO9Bwt1hFr5jIWW3lE1rmSJC3l/lkfdXefT/Zz9HHzxKy4YWfEVOhILbgqSXpU96g5EoFmzjOrv+dRfN32um6I1FLP3cUrLPyp7R5acvSmfJJ5dQdVdV7Pi74YYGQq+EWHjtQha/fzG563JndJ2TZV1xM+dpWGn+QTOhHSEAKu+o5Jw/nsOa/14z46GONxxqgJU/WEnpu0tpe7SNP6//M1vWb6HtF20JW/fpWLDNnKWqtP2yjT+d9Scabmig+fvNgPPBV96FebNaS/aZ2dTcX8OFTRey/L7lqCqDHYMA9B3sY9sbt3Hoq4cIbgvOyukz64qbOanj1x3s//R+el7oIbM2k9WPrKb4rRO+qzFhfHk+ym8tp/zW8liA+5v66d3ZS8cTHQD4F/opeF0BS+5eQmB5Yj5Zt2CbOSPcG8aT6UFEOPbsMQZaBqj9Xi0l15fg8aVe53O4q553QR7nN55P38E+up7touvZLo799hjegHM+vOXHLRyrP0bBZQXkX5JPWnHatNc9b4Ld+UwnvTt6EZ8gfkF8gifTQ8m1JQB0b+5msGUwNk38gjfgJefcHMDpTkVORJxpaQKDyXw188tAywBN32yi6f4mVm1YRdGVRVR9uoqlX1iakoEeT0ZlBqU3llJ6YymqGgt+34E+Wn/WytEHjwKQdXYWhVcUsuyeZaOO4yfD1cGOf/Naf9xK80PNo6b7CnyxYB/84kHaf9E+anp6RToXHLwAgJ0376Trqa6RiR54qe4l1j67FoCuZ7vwZnsJrAzgy3P12zprQq+GOPSVQ7RsaEEHlKI3FpFW4uzNYleLzVHxga26s4qKj1QQ3BKk6zfOHr37+e5Ym90f3j3p5c/td+cU+g710XBjA2XvKWPh2xay4psrqP5SNTqk6JA6d/DE3cRTfU81lR+tRAdHpnv8I3uDqjurWHT9Imdaf4TG3zWSX5sfm974L42c2OVcrJBWmkZgVYCiNxRRcXsFAANtA/gX+Ke8BZ5vNKy8fOXLDLYPUvruUso/UE6gJnWv9Jouj89D7vm55J6fS9UnqmLH56pK396+SS/PdcFWVVo2tLDr1l3okLLoHxcB4A14Y8c0Y8msziSzOnPc6fl/kz/qeeOKRpbULYk9X/PEGnp39BLaEaK3oZfeHb30HXD+QzSibFyyEfEJgZUB59+qAAWXFpC7PrnnO1NFZCBC689aaf1JK2c+fiaeNA+rf7qazJpM0hZM/5hzrhneAYgIZz52Jkxyf+CqYA+0DdD43kbaH2sn76I8Vj608pRhnUmBmgCBmgAL3rTgL6bpkFJ9bzW9DU7wu57touXhFiKfipC7PpehniFeveZVCi4roODyArLOzJo3e/bBrkGOPnCUw/cdZuDIAIHVAfoP9ZNZnTnrp6zcxFXBPvbcMTqe6GDZvcuo+GAF4k2NcHjSPCx+3+JR44a6h9Bw9HTIwX769vex50N7AKcrX3BZAZV3VJK1OjW/NXMm9O7sZfO5m4mEIuRfmk/td2spvLJw3mzUEmnOB3uoe4juF7opvKyQhW9fSO75uWRUZSS7rNPy5Y689VlnZHHeq+fRd6jP+fDk6S46n+yk4sPO8XnnM510PtVJ4RWF5F2chzcz9W8bHNer0NLUQsk7S8isyaT81nKKrykmZ21OsitzlTkd7K76LhpuaGCoY4jXHnwt/gL/nAj1eDIq4k6HRDR2XBV6OUTT15s4/OXDSLqQf3E+BZcXUH57+agP+FJRcFuQY/XHCL4YpOdPPbAd9lbtZeG1CxGvsOwLy5Jdoiul9l/FOMInwuz+4G62XrIVT5qHs54+C39BatxVM1PEI7EuacWHKrio6yLWPLmGxbcsZqB5gKb7mxCfM73p/iaOPnSU/qbkfZVPf3M/HU92cODzB3jl7a8QDoUBaPlRC7tv203HEx2klaXB+2H99vUpc5jkVnNujx0+EWbL+i30vtJL2fvKqL6nek58o8V0eQNeiq4soujKIgCGgkOx4B/5zhFCLzs3QARWB6AYjrzzCGX/7Nyq2PZYG748H/4FfvzFfvwL/HjSprZN14hyYu8J0krS8OX4aPtlG7tu2cVA80CsTcayDPqP9BNYEaD8NufyyrSyNESE+vr6OX8Oei6YM+/w8MUm3kwvJe8oIWddDoVXFCa7rKSJD8e6F9cR2hai85lOjj17jN5Xeund2Qs4QXzlba+MOmcPUP6hcpZ/aTmRgQjb37KdtOK0WOj9xX5yz8sl64wsBjsHaX+8neBLQYIvBgluDRLuCXPGY2dQfHUxGZUZFLy+gJxzcshem0322uxRF+ikl6XPyvthRpsTwQ41hGi4oYHqL1WTf1E+VR+vSnZJKUU8QvbZ2WSfnU3lhyupr69ned3I1/qs37aewbZBBtsHGWgbYLB9kJx1zodV4WCYgaYBQltDDLQNoP3OJ/VLv7DUCXbHIDtv2okny0P22dmUXF9Czjk55Kx35s85N4dVD62a/RdtTikhwRaRfOAmoAV4PXA3kA1cCoSAXar6zESWdfhrh9l7x148WR7Cx8OJKNfVxCOnPGXmL/Sz7sV1gNMrCofCDLYN4s12Dm8yqzM5r+E8Mpdn2nHxHJKoPfYq4GlV3SYiTcDNQAVwnapGROQnIvJbVT11Ug/B7g/spvCqQmofrCW91Lp1iSQi+LJ9o7r54hECte69lNOtEhJsVX0+7mkRsA1YpqrDR3qdOEHff8oF9UHNgzWU3lRqFy0YMwkJPcYWkQBwMfAVnG74sONAMWMEW0RuxtnDU1JUQuPyRhr/tzGRZU5JMBikvr4+2WWMyWqbmlSubdJUNSH/cC6v+ARQCKQBv4yb9i2g8nTLqKmp0VT13HPPJbuEcVltU5PKtQGbdRL5S+QFKjcBD6lqp6oOAAMiMnzCuQBoSuC6jZnXEvWp+FuA64CLosfG/cAXgY+JSBD4rp7ugzNjzJQl6sOzx4DHxpj0YiLWZ4wZbU5eK26MOTULtjEuZME2xoUs2Ma4kAXbGBeyYBvjQhZsY1zIgm2MC1mwjXEhC7YxLmTBNsaFLNjGuJAF2xgXsmAb40IWbGNcyIJtjAtZsI1xIQu2MS5kwTbGhSzYxriQBdsYF7JgG+NCFmxjXMiCbYwLWbCNcaEpBVtEzpvpQowxM2fcn/gRkVuAqwE9eRKwCFiTwLqMMdNwqt/uCqrq5WNNEJGrE1SPMWYGjNsVV9WH45+LSIGIrBWRXFX95WRWItGf3DTGzI4J/dqmiNwMXA7sAapF5FequuE083iAdwB/D/wH8EcRuQG4KK7Zp1XVfifbmBk20Z/RrVTVtw0/EZHPn24GVY0AG0RkOc5x+fD4f5p0lcaYSZlosBtPer4PQESWquq+SayvN/qh3ELgeVV9ahLzGmMmaKLBXicilUAE57h8tYgUA6uBd010Zar6c4h1038sIrtUdW98m2i3/2aA4uJi6uvrJ7r4WRUMBq22KbDaZsdEgx0Gfhf3/PfRx5NPhU2IqkZE5DngXGDvSdMeAB4AqK2t1bq6uqmsIuHq6+ux2ibPapsdEw3294GPA4VAM/BJVT0gIn+a6Iqin4zfoqrfjI5aBvxxMsUaYyZmosF+L3CrqraJSAnwKeB9qto73gwiEgCuwtkrZwBpQFBE3gAsBnar6vZpVW+MGdNEg71VVdsAVLVFRHYCiEi6qvaPNUM09I9E/xljZtFErxXvFpFyESkTkcXAkIiUAdcksDZjzBRNdI99C84FKvFXkK0HKoGHx5zDGJM0Ew327aq6+eSRIrJihusxxsyACXXFxwp1dPyumS3HGDMT7IsWjHEhC7YxLmTBNsaFLNjGuJAF2xgXsmAb40IWbGNcyIJtjAtZsI1xIQu2MS5kwTbGhSzYxriQBdsYF7JgG+NCFmxjXMiCbYwLWbCNcSELtjEuZME2xoUs2Ma4kAXbGBeyYBvjQhZsY1xoVoId/aVNY8wsSViwRcQjIu8SkceBC6PjlorIXSJyo4hcm6h1GzPfJSzYqhpR1Q3AS4z85tddwH2q+n3gUhHJT9T6jZnPZvsYu0RVu6PDjcDaWV6/MfPCbAc7LW74OFA8y+s3Zl6Y6K9tzpSBuOECoOHkBiJyM3AzQHFxMfX19bNT2SQFg0GrbQqsttkx28FuFpE8VT0OrAC+dXIDVX0AeACgtrZW6+rqZrfCCaqvr8dqmzyrbXYkLNgiEgCuAs4FMkQkDbgbuF1EmoDfxB1vG2NmUMKCraq9wCPRf/E+k6h1GmMcduWZMS5kwTbGhSzYxriQBdsYF7JgG+NCFmxjXMiCbYwLWbCNcSELtjEuZME2xoUs2Ma4kAXbGBeyYBvjQrN9P/as6ekbZPOBLjbt7WTTvg5ajvexoiSHVaW5rCp1HpctyMLntW2bcR/XBPt47yAv7O9k094ONu3r5JUjx4ko+L3CWeX5rF9aSGNLkP/bs5fBsAKQ5vNQU5LNykW5TuAXOYEvyEo7zdqMSW1zNtidoQFe2NfBxr2dbNrXSUNzN6pOWNdW5PP+S5Zz/rIiXlNZQGaaNzbfwFCEve1BdhztZsfRHnYc7aZ+ZxuPbjkca7MoN4OVpcN7dyfwS23vbuaQORPs1p4+XtjXGetaN7YEAcjwezi3qoDbL6vhvKWFrK3IJ8PvHXc5aT4PKxflsnJRLlefMzK+raefhubuUYH/w652hiLO3j3d56GmJIeV0b16e/MQ0thGToaP3AwfORl+stN9BNK82O8jmGRL6WCHBpU7H9vGpn0d7G0LAZCV5uXcJYW8ae1iXruskDWL80nzTX9PWpyTTnFOMRevGPni1IGhCLtbg6MC/9uGVh6J7t3vf+mFv1iO1yNkp/vITvdFQ+8nO8MZzsnwkZ3uj20MsjN85KT7CaR78Xk8eD3gEcHn8eDxOMvyijiPHsETNzw8zRMd9sVNV9Vpvx9mbpNU/iNIL12hNe/5BuuXFHL+0kLOX1bEmWW5Se8St/X081T9H1m5Zi09fUP09A/R0zdIT98Qwb6R4fjxPX1DBKPPh4/xE8kjzkbC4xE8grMRiH8e3RAMbwxk1Dhi4515nOcCSNx0jzjzxbdx1j3SZlR7D7S1tbFw4UKA2PKcx9HPERAkbjzReaLjoo2GpxNbhsQND48f6UGN3wYOHTpMZWXFqPXHt/uLGhgZKWMs2yMj6x55r+Jfw8hrH57+F+Oi815zXuUWVV030f//lN5jl2V5eOlTV+D1pFbXtjgnnfIcD+uWFE56XlWlfygSDbsT+tDAEJEIhFWJRJShiBKOKBEdeRwKa2x6ODo+PKodhCMRwhHYu28flVVV0WkQiZtPFWc+VTS2fIgMLyd+OPqow8tQ0OhrcJZJrN1gWE/ZRtV5faFQhM5wNwoQ19Z5BMVpO7y/GZ7GydMZbqNx08aZJ7qwuOax9Y4MQzgSxtO0f9SyiFtn/Hwj609NKR3sNC8pF+rpEhEy/F4y/F6Kc9ITso76+ibq6moTsuzpSuWv+J1Obapjhz8SNz5+IxnbkOnIhmp4evyGY7ht+T2Tqyelg23MXCHRrnTcmGSVAtiVZ8a4kgXbGBeyYBvjQhZsY1wopc9ji0gPsDPZdYxjAdCe7CLGYbVNTSrXVquqORNtnOqfiu+czEn52SQim622ybPapkZENk+mvXXFjXEhC7YxLpTqwX4g2QWcgtU2NVbb1EyqtpT+8MwYMzWpvsc2LiMiHhFZJyJnJ7uWuUCmeHN/ygZbRN4vIu8UkU+KyORvo0oQEckXkQ+JyLtE5IciUpPsmuKJSK6I/DrZdYxFRBYBHwXaVHVrsusZJiJeEfmIiPyDiHxVRCqTXI8n+vf1OHBhdNxSEblLRG4UkWtPt4yUDLaIrARKVPVHwM+B25NcUrxVwNOqugH4L+DmJNcTE926vxVoTXYtJ4vW9ingq6p6INn1nGQN4FfVHwI/Bq5OZjGqGon+fb3EyN0kdwH3qer3gUtFJP9Uy0jJYAMXA1sAVHUn8JrkljNCVZ9X1W3Rp0U4b36qeCPwJBBJdiFjqAMGgRtE5G4RKUlyPfEagItFZDFwDvA/Sa5nLCWq2h0dbgTWnqpxqgZ7AdAd99yfrELGIyIBnA3QI8muBUBEVgPHVLU52bWMYy3woqp+G3gU+GqS64lR1T7gbmADsAzYl9yKxhT/1bnHgeLxGkLqBrsVyIdYF24gueWMFq3pduCzqtqf7Hqi3gBcKCJ3AGtE5I7oxidVDMUNvwxUJKuQk4nIKuAsVb0EeBa4N8kljSU+AwVAy6kap2qwfw8MX9q3GpjU5XSz4CbgIVXtTHYhw1T1XlX9oqp+EdgWHe5Ndl1xfs/IIdVCTvOHOcuuwOmOAzxDCm104jSLSF50eAWnOQRMyWvFVbVRRI6IyA3AEuBrya1ohIi8BbgOuCh6JqJfVd+T3KpSn6q+JCIHReRGoBS4M9k1xfkp8Inoh7aFwLeTWUy0p3UVcC6QISJpOIcKt4tIE/CbuOPtsZdhF6gY4z6p2hU3xkyDBdsYF7JgG+NCFmxjXMiCbYwLWbDNtIhInYik4gUd85oF20yLqtaTQtfyG4cF2xgXSskrz0xiiMhtwDGgCvgdzu2JXpw76VYAB1X1OyKSDdwG7AFKgE2quklEinFuU20EqoEHVbUjuuzX49wZ9XfAlae7MsoklgV7nhCR1wGZqnpf9CaWh4HvAX+tqg9F2zwqIo8CHwR+oqrbo21/JSJvAr4AfFxVW6O3OC4COnB6fltV9SkRiQAXAE/N9ms0IyzY88fwdcdvxtlLD3/ZQfy923tw9uZnA58BUFUVkRNAHlClqq3R8U1A0/Ay4m4XbeU0txSaxLNj7PljB9Cjqo+r6i+IBpfRv/daDewFtgE1ELtFNRPnHuB2EVkx3FhEcmehbjMFdhPIPBEN6L/h3LB/GCe8AO8FnsO5i26Hqj4cDextOCEvATaq6vPR7wL7PLAL6AR+BlQCPwBuxbmV8N+BMHCnqvbMzqszJ7Ngz2MiUgdcpKqfS3YtZmZZV3yeEpEsnG/AXC0i1cmux8ws22Mb40K2xzbGhSzYxriQBdsYF7JgG+NCFmxjXMiCbYwL/T/nTalD/TPz8wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.0004, 10, lang, device, first_train=False, lloss=1.912,lppl=6.844)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAC4CAYAAAAsVToeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAGZxJREFUeJzt3XucXHV98PHPdy57mdn7PYRNstkkm2SBJMArJkBkEVtoq7TSPi9KxacoFKsPRaIVFRWhVLT1VYVKi1IvyK2PMVK11SACWdDHoCQkIUAukE2M5LK72fvsfWe+zx/n7O5ks/fs7OzkfN+v17zmXOd8J9nvOb8553cRVcUY4z2+ZAdgjEkOS35jPMqS3xiPsuQ3xqMs+Y3xKEt+YzzKkt8Yj7LkN8ajLPmN8ShLfmM8KpDsAM5EXl6eLlmyJNlhjKqzs5NwOJzsMEZlsU3PXI4NYMeOHSdVtXiy26d08peWlrJ9+/ZkhzGq2tpaampqkh3GqCy26ZnLsQGIyO+msr0V+43xKEt+YzzKkt8Yj0rt5G+Dvoa+ZEdhTEpK7eRvgN1/sJv+5v5kR2JMyknt5J8PXfu6ePWPXmWgfSDZ0RiTUhKS/CKSLSKfFZG/EpGHRWTJiPWFIvIFEXm/iHw0bvmt7rLPi0jBhAcKQfUPqom8EmHPe/YQ7Yom4NsYc3ZK1JW/GHhRVZ8EHgU+PGL9J4AnVfUJYL6ILBWR5UCpu2wTsHEyByq6pogVj6+gfVs7Lc+3zOBXMObslpBKPqpaB9S5sxXAiyM2uUBV33SndwGXAkFgh7v/fhG5cLLHK7muhOy12WRWZJ5Z4MZ4SMJ+84tImYjcD6wAtoxYnRY33YZTUigC2uOWB6dyvMHEb36mmX0f2odGrVdiY8aTsOq9qnoCuF1EFgJf4dRifPzzuXygHifZ8wBEREZsM0REbgFuASguLqa2tvbUDX4EfBdOHD8BnyRptzQjkcjpsc0RFtv0zOXYpkVVZ/wFXAic604XAM/jXO3T3GX3Asvd6S8Di4FlwH3usmrg7omOs2zZMh1N3V11upWteuDWAxqLxUbdJtG2bt2alONOhsU2PXM5NlVVYLtOIU8TdeU/CtwgIk3ARcCdwG2A4JQCvoZTKjgIHFHnHgEickxEbgQWAQ9M9+CL7l5EtDPK2//yNr6wj8VfWoxTmDDGDErUDb964F/c2Ufc95fi1jcDd42y34MzcXwRofIrlcS6YvSf7AfFOe0YY4akdJPe8YgISx9c6kz7hGhnFH/Yn+SojJk7UruG3wTEJ4hP6D3ay8vnvcyxbx5LdkjGzBlndfIPChYHCZ8X5sBHDnDisRPJDseYOcETye9L87HyByvJe1ce+27cR8PmhmSHZEzSeSL5AfwZfs7/8fnkrM9h7/V7E1oVuOmnTfAQNGxusCbHZs46a2/4jcYf9nPBTy/g4B0HyVqTNSOf2X2om5bnWmh9vpXKr1SSPj+dnsM9sBne2PQGAKEVIfIuz2PJ/UvwpXvmfGvmOE8lP0AgN0DVN6sAiPZE6T7QTdYFUzsRdB/q5sh9R2h5roWeQz0ApJWl0V3XTfr8dM75yDm8WfUma7LW0PZCG60vtNL2/9qGEv/gJw8y0DpA7uW55F2eR0Z5xsx+SWMmwXPJH++t296i4fsNrHpuFTkX54y6zUD7AK0vtNLyXAu5l+VS8hclSEBo3NxIXk0e5378XPKvzCe0PDRUkUh8AgHIXZdL7rpcFnxqwWDNRwD6W/pp3NzI8W8dByCjIoOyvy5j0RcWJfw7GzPI08m/8PMLaX6mmVevepXVtavJOt8pAagqh79wmJZftND+cjtEwZfhI5gfhL+AjPIMLj15KeKffM2h+BqGy7+1nKpvVhHZExkqGQw2RIoNxNi+ajtZa7LIuzyPvMvzyFyaaTUUzYzzdPJnlGew+vnV7Nywk93v3s28m+ex+ItOVeDmLc1IQFjw6QXkX5lPzvoc/BnDlYSmkvijEb+QvTqb7NXZnPuxc4eWR9uihKvDtPyihYYnnKcSaWVpVH61ktLrS8/omMbE83TyA2QuzmTVc6t4/X2v07q1FVVFRFizbQ2+wOzfnAsWBqneVI2q0rW/i7YX2mh5voXMSqfJcsfODpp/3kzZB8pIn58+6/GZs4fnkx8gvDzM2r1rT1mWjMSPJyKEl4cJLw9zzofPGVre8osWDn3mEIc+e4j8P8in7MYyiv60CH+mVV02U2PPnVLMgjsWsPbNtSy8cyFde7vYe/1eXq5+GY3NfuclGlWiPdZvYqqyK38KCi0JUXFvBYvuWURrbSu9b/ciPkFV2fOePeRuyJ3xnwUaVTp2dhDZFSGy0329GqHi3grKN5bT39LP0QePUvBHBWRfmO088TBzmiV/ChOfkP+u/KH5geYBoh3RM/5Z0N/a7yT5rgjp89Ipua6EWG+MV97xCsTAn+0na3UW826aR/bF2QB0/LaDw3cd5vBdhwkWBym4qoCCqwsofG8hgRz7M5uL7H/lLBIsDLLmxTV0vdVF/aP1nPjeCfZev5fqzdUU/3kxsb4YEhy+Iqsq0fYogVznz2Dfzftofa7VqaHoKv5fxZRcV4I/5Of8n55PaGmIjIqM067sBVcVcEn9JTQ/00zzlmaatjRR/3g9a/etJZATILI7QrQrSs7anDN+UmJmhiX/WSi0JETFP1Sw6O5FtG5tJfeyXACOfPkI9U/WwwrYfd9uIjsjBEuDrH3NvdmpkL02m3m3zCNrTRbZa7JJKx3ua7Xw6sJxj5tWkkbZDWWU3VCGRpXIrgiZy5ynFEe+coSGJxoI5AfI/8N8Cq52SgbpZfbEIlks+c9i4hPyrxz+WRA+P0xaaRrdP+um/7x+Cq8pHCq2Ayz/9vKZO7ZfyL5o+LOXPrCUovcW0bSlieanm2n8fiOZVZm8Y987AOjc1zn0OHM0GlOikSgD7QNE26NEO6KknZNGRnkG/a391D9WT7TDXd8RJdoepfSvSyl4dwEDHU4tzfDKMBkLM6zk4UpI8otIHnATTq+8VwH3quqBuPX3A4MV6ouA76jqT0TkKaDZXb5DVR9KRHxeVfy+Yorf5/R4fHHNxbN67GBhkJLrSii5rgSNKZHdEaeLNZxajTsv2enUclwGu7J3Ee2IUnpDKed+7Fz6Gvv4dcmvT/vMii9WsPDOhUTborx121sASFDw5/gJZDslDIDIrgivvfc1wKmpGVoeIrQiRPkd5WSvzibWGwMf+ILeeviVqCv/CuAZVd0jIkdxutr++8GVqnr74LSI/B3wtDv7E1V9JEExmTlCfEL2muFSAQpV36qieUszx184Tiw9RrA4iD/XuUkZyA2w8K6FBHIC+LP9TnLnBAitCAGQfm46lzReQiA7MGqryeyLslnz6zV0vdFF5xuddO3tou3Xbczvng9A4381su9/7yNzaSbhlWFCK0KEVoYo/OOz+2Zlojrw3BY3W4gzKs9pRCQEZKrqYKP3ahHZCISBr6tqWyLiM3OLL+ij+Npiiq8t5njtcS6sOXWwJl+aj4p7KsbcX/xCWlHamOv9IT+563PJXZ876vrQ8hDlnyinc28nkd0RGp9qhBisO7KOQE6A4989zsn/OglZcPzwcafksDxEMG9K48rMORLf2mzGP9xJ7i8Bd6hq7yjrPwI8q8NDdw0uXwN8VFX/ZpR94gftuGjTpk0Jif1MRSIRsrJmps+AmWaxTaAP+D3OaBIC/Bj4Eejbigy49wt8OOXVILAVaAIWuK8SklJ97oorrtihqpP+PZew5HdH3bkTeMjtqnvker+77pYx9n9WVd893jGqqqp0//79MxLvTKutraWmpibZYYzKYpue2udqWbtgLV37uug73sc5tzjVrl+79jWnZODyZfrIfWcuq55eBUDrr1rxZ/kJLQvhDyWuGraITCn5E/mD5ibgkcHEF5E0gLgi/jU4I/ngrr8CaFDV10UkH7CeNs3c4ofQ0hChpaFTFlf/sJr+k/107esaesXfezjw4QN0vdEFQPrCdELLQxRcXUD57eWzGv5Iibrbfy1wPXCZ2w69F3iT4RF7AD7gbjNoJ/AhETkPWI5TajBmzhMR0orTSCtOI29D3mnrqzdV07m385STQ/eB7iREeqpE3fB7Cnhqgm2uHTHfCnw1EfEYk0zh6jDh6nCywziNtx5sGmOGWPIb41GW/MZ4lCW/MR5lyW+MR1nyG+NRlvzGeJQlvzEeZclvjEdZ8hvjUZb8xniUJb8xHmXJb4xHWfIb41GW/MZ4lCW/MR5lyW+MRyVr0I4a4CNAh7vo66q6W0RuBVpw+k39t9E6/jTGzIykDNrh+pSqHh6cEZHlQKmqPigiVcBG4PMJis8Yz5tU8ovIB4HfAH7gc8CW8UbWmcSgHb3AX7mdex5W1SeBDcAOd//9InIhxpiEmeyVv01V3xCRx3GK8zdMZid30I4NwB3xy92TwzZ3m6+JyO9xxuw7GLdZag+HYswcN9nkzxWRdcARVY2ISNNEO7iDdmwE7hlttJ44TwPrgAYgL27fvtE2HjFiD7W1tZP8CrMrEolYbNNgsc0iVZ3wBVwI3A3kAquB2yaxz83A/Lj5NCDNnf4/DI8W9FHgj4FlwH3usmrg7omOsWzZMp2rtm7dmuwQxmSxTc9cjk1VFdiuk8jnwdekrvyq+grwCoCIdALfGG/7SQzacQS4VkRycAbl3KKqKiLHRORGYBHwwGRiM8ZMz2Rv+N0NbAIuAVYCnYxzJ14nGLRDVf97jOUPTiYeY8yZm2wln1eBfcDVqvpx4K3EhWSMmQ2TTf4K4OvAJhHJArITF5IxZjZM9m7/14AFqnpYRJYALycwJmPMLJjslV+By0Xkq8ClwG8TF5IxZjZMNvn/Hqce/sNAG/DJhEVkjJkVky32/969gw+wT0Ten6iAjDGzY7JX/qwR83NvsHFjzJRM9sp/XEQewamcU844z/CNMalhsjX8fioivwKW4DS+2ZDQqIwxCTdm8ovIP+HU6deRq4AyYNRaesaY1DDelf9nqvqp0VaIyPoExWOMmSVj3vBT1RfGWbdtrHXGmNRgHXga41GW/MZ4lCW/MR5lyW+MR1nyG+NRlvzGeFSyRux5N1AMLADmAx9z+/B7ChgcpWeHqj6UiPiMMckbsWcB8KiqDojI93B6630N+ImOMxiIMWbmJCT5dYIRe1T1O3GzaUCdO10tIhtxWg1+XVXbEhGfMWa47/zEfLgzYs+XgDt0lIE7RORyYImqfnvE8jXAR1X1b0bZJ37Qjos2bdqUkNjPVCQSIStrZEvoucFim565HBvAFVdcsUNVL570DlPp5H8qL5wGQJ8FCsZYvwC4fZz9n53oGDZox/RYbNMzl2NTnfqgHYm8238T8Ii6w2yLSJqIpLnTBcA1xA3MISJXiEi1O50PnEhgbMZ4XqLu9k80Ys9D7rKL3fXP4TQR/pCInAcsB+5MRGzGGEeibvhNNGLPdWOs+moi4jHGnM4q+RjjUZb8xniUJb8xHmXJb4xHWfIb41GW/MZ4lCW/MR5lyW+MR1nyG+NRlvzGeJQlvzEeZclvjEdZ8hvjUZb8xniUJb8xHmXJb4xHWfIb41HJGrSjELgVeAvIVdV/d5ffCrQAi4F/G+z/zxgz8xJ15R8ctONx4Du4XW3H+QTwpKo+AcwXkaUishwodZdtAjYmKDZjDEkatAO4QFXfdKd3AZcCQWCHu/9+EbkwEbEZYxyJGq4LGBq0YwNwx4hVaXHTbcAiN5aDccuDY3xm/KAd1NbWzlC0MysSiVhs02CxzZ6EJb84fXJvBO7R00fr6Yubzse5NxAE8uL27WMUqvow8DBAVVWV1tTUzGzgM6S2thaLbeosttmTlEE7gJ3ub3yANcCvgF8Cg0MNrQS2JzA2YzwvWYN2fA24XUQOAkdUtc7d75iI3IjzM+CB0z/ZGDNTkjVoRzNw1yjLH0xEPMaY01klH2M8ypLfGI9K6KO+uaqzd4BDJzupO9lJXWOEusZOjjR3UV4QYv3iQtZXFrKoMIR7v8KYs9JZm/wD0RhHW7upa+zkYGPESfbGTupORqhvH37yKALn5GZSXpDJS3VN/PfuYwDMy81g/eJC1lUWsn5xIeUFoWR9FWMSIuWTvynSS93JTg41dnLwZIRDjc4V/XdNnfRHdWi7nIwAi4uzuHRJEZXFWVQUhVlcHGZRYZiMoB8AVeVgYyfb6pp46WATLxxo5KmdRwE4Nz9zqFSwvrKQebmZSfm+xsyUlE7+Ix0xLvrHZ4fmg35hYWGYiqIwV64oobIoi4riMIuLwhSE0yYsxosIS0qyWFKSxQfWLURVOVAfYdvBk2yra+KZN+r5wY63AVhUGGJ9ZRHrKwtZt7iAkuyMhH5XY2ZaSid/OCB87k9WUFmcxeLiMPPzMgn4Z+4epohQVZZNVVk2N15aQSymvHG8nZfqmth2sIn/2X2M//ztEQCWlGQNlQzWLS6csRiMSZSUTv7CTOHmDYtn7Xg+n3De/FzOm5/LzRsWMxCN8fqxdra5J4MfvvI2j730OwDKQkLlWy9RmpPhvLLTnfdcZ744K520gD1sMcmT0smfbAG/j1Xleawqz+NvL6+kPxrj1bdb2Xawia27D9LVF+U3dc3Ut/cwENPT9i8Mp1GSk0FZjnNiKMnJoDQnnTL3hFGSk05hOB2/z546mJlnyT+Dgn4fFy0s4KKFBZznO0pNzaUAxGJKc1cf9e09NLT3cqK9h/r2Hurbe2lo7+FEew97jrbT1NmLjjhH+H1CcVY6OZkBMoL+oVdm0Oe+++OW+4bmM4N+MtL8ZAR8ZKbFLQv6aOyKcbytm4DPR8An+P1C0OfD7xMCPsFnJxtPsOSfBT6fUJSVTlFWOtXnjL1dfzTGyUgv9e29nGjroaFj+CQR6RmgZyBKd1+Utq4+6vtjdPdH6emP0t0fpbc/Rl80NvmgXnx+7HgF58Tgl6ETQsDvnih8QtDvi1su+MU5YcS/+92TSMAn+ETw+5wTmc9dd9o+PvCLcOxYLy92vDGFf93JGbzXKyPn3Ymh093QdnLafkeO9LG9dz8+cfbzieAT5/9XBHzi7OWT4fnh9c604C4ToSw3g3cuK57x7zpZlvxzSNDvY15upvMYsXzq+0djOnQy6Bl6xZ0k+qL0DMTYved1liyrYiCmDERjRGM6ND0QU6IxpT+qRGMxd7m6y2Nx00p/3PYxdd6dz4rRO6BE1Sn1DMSUWEyJ6vB7NBY/zdD+ff0DBE78fkb/XQcLU+oWq4bnB9friPlTdxxcH40pHHoLjdv2TLxzWbElv5kZfp8QTg8QTh//vzWn5QA1axfMUlRTM5fbzMfHpqqoOietmPs+OK+477HB9c42OviO8542g0+mpsOS35hpkMGiPal7f8SeNRnjUZb8xniUJb8xHmXJb4xHic7EM4skEZEOYH+y4xhDEXAy2UGMwWKbnrkcG0CVqmZPduNUv9u/X1Uvnniz2Sci2y22qbPYpk9EptTjtRX7jfEoS35jPCrVk//hZAcwDotteiy26ZtSfCl9w88YM32pfuU3ZxkR8YnIxSKyKtmxpAqZZjfTKZv8InKriLxfRD4vIgXJjmeQiOSJyCdE5AYReUxEliU7ppFEJEdEnk52HCOJSBnOiM6Nqro72fEMEhG/iHxSRD4gIveLSNJbRbknyRtE5EfAJe6yChH5nIh8UET+cqLPSMnkdwf5LFXVJ4BNOKMBzxUrgGdU9XHgO7jDic8V7lXiz4GGZMcSz43rLuB+Vf1dsuMZ4XwgqKqPAU8C70tyPKhqzP0b28VwdwSfA/5VVb8LXCkieeN9RkomP7AB2AGgqvuBC5MbzjBV3aaqe9zZQpz/nLnkvcAWYAo9f8yKGqAfuFFE7hWR0iTHE28fsEFE5uOMKv2zJMczllJVbXenDwCrx9s4VZO/CGiPmw8mK5CxiEgI5yT1g2THMkhEVgKtqnoi2bGMYjWwU1W/AWwG7k9yPENUtQe4F3gcWAwcSm5EY0qLm24Dxu0pJFWTvwHIg6HiYl9ywzmVG9NG4B5V7Z1o+1n0HuASEfk0cL6IfNo9Sc0FA3HTrzKtvowSQ0RWABeo6hXAc8A/JzmkscTnQT5QP97GqZr8vwQGq1muBKZUrXEW3AQ84g5FPmeo6j+r6pdV9cvAHne6K9lxuX7J8M+3Eib4w51lf4hT9Af4BXPoxDTCCRHJdaeXMsFPzpSs26+qB0TkmIjcCCwCHkhuRMNE5FrgeuAy9wlMr6p+OLlRzX2quktEjojIB4F5wGeSHVOc/wt81r3RXAB8I8nxDP6s/BPgIiBDRNJwfppsFJGjwLNxv/9H/wyr5GOMN6Vqsd8Yc4Ys+Y3xKEt+YzzKkt8Yj7LkN8ajLPlNwolIjYjM1YoxnmXJbxJOVWuZQ+0vjMOS3xiPSskafiZxROQ2oBVYCLyI03zVj9OKcilwRFW/KSJZwG3AQaAU+I2q/kZEinGaMR8AKoH/UNUm97OvwmkVdw1w9UQ10ExiWfKbISLyLiBTVf/VbZz0KPBt4J2q+oi7zWYR2Qx8HPhPVX3N3fbHIvKnwJeAO1W1wW0CWwY04ZQyd6vqz0UkBqwHfj7b39EMs+Q38Qbrif8ZztV+sFON+Lb/B3FKBauAuwFUVUWkG8gFFqpqg7v8KHB08DPimhI3MEFzU5N49pvfxNsLdKjqj1T1h7jJDaeMQ10J1AF7gGUw1IQ5E6cN+UkRWTq4sYjkzELcZhqsYY8Z4ibxP+B0CvE2ToID/C2wFacF5V5VfdRN6ttwTgSlwEuqus3t3+4+4E2gGfg+sAD4HvB3OM1MvwhEgc+oasfsfDszkiW/GZeI1ACXqeo/JjsWM7Os2G/GJCJhnJ5hV4pIZbLjMTPLrvzGeJRd+Y3xKEt+YzzKkt8Yj7LkN8ajLPmN8ShLfmM86v8DdFmKddqrz7kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_seq2seq(net, train_iter, valid_iter, 0.0005, 10, lang, device, first_train=False, lloss=1.898,lppl=6.744)\n",
    "torch.save(encoder, MODEL_ROOT + \"trans_encoder2{}.mdl\".format(num_examples))\n",
    "torch.save(decoder, MODEL_ROOT + \"trans_decoder2{}.mdl\".format(num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": "400"
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
